@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 14f91fe67c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Spec Decode] Disable Log Prob serialization to CPU for spec decoding for both draft and target models. (#6485)

sroy745 committed Jul 20, 2024 at 23:58 UTC 14f91fe67c2342f2fe859dc6a5c40810df0e1c61
8 files changed +332 -63
tests/spec_decode/e2e/test_logprobs.py
+29 -19
@@ -22,10 +22,12 @@ from .conftest import get_logprobs_from_llm_generator
22 }])
23 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
24 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
25 -@pytest.mark.parametrize("test_llm_kwargs", [{
26 - "speculative_model": "JackFram/llama-160m",
27 - "num_speculative_tokens": 3,
28 -}])
25 +@pytest.mark.parametrize("test_llm_kwargs",
26 + [{
27 + "speculative_model": "JackFram/llama-160m",
28 + "num_speculative_tokens": 3,
29 + "disable_logprobs_during_spec_decoding": False,
30 + }])
31 @pytest.mark.parametrize("batch_size", [8])
32 @pytest.mark.parametrize(
33 "output_len",
@@ -59,10 +61,12 @@ def test_logprobs_equality(baseline_llm_generator, test_llm_generator,
61 }])
62 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
63 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
62 -@pytest.mark.parametrize("test_llm_kwargs", [{
63 - "speculative_model": "JackFram/llama-160m",
64 - "num_speculative_tokens": 3,
65 -}])
64 +@pytest.mark.parametrize("test_llm_kwargs",
65 + [{
66 + "speculative_model": "JackFram/llama-160m",
67 + "num_speculative_tokens": 3,
68 + "disable_logprobs_during_spec_decoding": False,
69 + }])
70 @pytest.mark.parametrize("batch_size", [1])
71 @pytest.mark.parametrize("num_logprobs", [6])
72 @pytest.mark.parametrize(
@@ -99,13 +103,16 @@ def test_diff_num_logprobs(baseline_llm_generator, test_llm_generator,
103 }])
104 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
105 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
102 -@pytest.mark.parametrize("test_llm_kwargs", [{
103 - "speculative_model": "JackFram/llama-160m",
104 - "num_speculative_tokens": 3,
105 -}, {
106 - "speculative_model": "JackFram/llama-160m",
107 - "num_speculative_tokens": 6,
108 -}])
106 +@pytest.mark.parametrize("test_llm_kwargs",
107 + [{
108 + "speculative_model": "JackFram/llama-160m",
109 + "num_speculative_tokens": 3,
110 + "disable_logprobs_during_spec_decoding": False,
111 + }, {
112 + "speculative_model": "JackFram/llama-160m",
113 + "num_speculative_tokens": 6,
114 + "disable_logprobs_during_spec_decoding": False,
115 + }])
116 @pytest.mark.parametrize("batch_size", [8])
117 @pytest.mark.parametrize(
118 "output_len",
@@ -143,6 +150,7 @@ def test_logprobs_different_k(baseline_llm_generator, test_llm_generator,
150 [{
151 "speculative_model": "JackFram/llama-160m",
152 "num_speculative_tokens": 3,
153 + "disable_logprobs_during_spec_decoding": False,
154
155 # Artificially limit the draft model max model len; this forces vLLM
156 # to skip speculation once the sequences grow beyond 32-k tokens.
@@ -181,10 +189,12 @@ def test_logprobs_when_skip_speculation(baseline_llm_generator,
189 }])
190 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
191 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
184 -@pytest.mark.parametrize("test_llm_kwargs", [{
185 - "speculative_model": "JackFram/llama-160m",
186 - "num_speculative_tokens": 3,
187 -}])
192 +@pytest.mark.parametrize("test_llm_kwargs",
193 + [{
194 + "speculative_model": "JackFram/llama-160m",
195 + "num_speculative_tokens": 3,
196 + "disable_logprobs_during_spec_decoding": False,
197 + }])
198 @pytest.mark.parametrize("batch_size", [1])
199 @pytest.mark.parametrize(
200 "output_len",
tests/spec_decode/test_dynamic_spec_decode.py
+1
@@ -32,6 +32,7 @@ def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int,
32 scorer_worker=target_worker,
33 spec_decode_sampler=mock_spec_decode_sampler(
34 acceptance_sampler_method),
35 + disable_logprobs=False,
36 metrics_collector=metrics_collector,
37 disable_by_batch_size=disable_by_batch_size)
38
tests/spec_decode/test_spec_decode_worker.py
+13 -7
@@ -381,6 +381,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool,
381 worker = SpecDecodeWorker(draft_worker,
382 target_worker,
383 spec_decode_sampler,
384 + disable_logprobs=False,
385 metrics_collector=metrics_collector)
386 worker.init_device()
387
@@ -479,7 +480,8 @@ def test_k_equals_zero(k: int, batch_size: int,
480
481 worker = SpecDecodeWorker(
482 draft_worker, target_worker,
482 - mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
483 + mock_spec_decode_sampler(acceptance_sampler_method), False,
484 + metrics_collector)
485
486 seq_group_metadata_list, _, _ = create_batch(batch_size,
487 k,
@@ -490,9 +492,10 @@ def test_k_equals_zero(k: int, batch_size: int,
492 out = worker.execute_model(execute_model_req=execute_model_req)
493
494 assert len(out) == 1, f"expected only one token output when {k=}"
493 - assert out[0].probs is None, "expect gpu tensor references to be None"
495 + assert out[0].sampled_token_probs is None, (
496 + "expect gpu tensor references to be None")
497 assert out[
495 - 0].sampled_tokens is None, "expect gpu tensor references to be None"
498 + 0].sampled_token_ids is None, "expect gpu tensor references to be None"
499
500 draft_worker.execute_model.assert_called_once_with(execute_model_req)
501 target_worker.execute_model.assert_called_once_with(execute_model_req)
@@ -524,7 +527,8 @@ def test_empty_input_batch(k: int, batch_size: int,
527
528 worker = SpecDecodeWorker(
529 draft_worker, target_worker,
527 - mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
530 + mock_spec_decode_sampler(acceptance_sampler_method), False,
531 + metrics_collector)
532
533 seq_group_metadata_list, _, _ = create_batch(batch_size,
534 k,
@@ -535,9 +539,10 @@ def test_empty_input_batch(k: int, batch_size: int,
539 out = worker.execute_model(execute_model_req=execute_model_req)
540
541 assert len(out) == 1, f"expected only one token output when {k=}"
538 - assert out[0].probs is None, "expect gpu tensor references to be None"
542 + assert out[0].sampled_token_probs is None, (
543 + "expect gpu tensor references to be None")
544 assert out[
540 - 0].sampled_tokens is None, "expect gpu tensor references to be None"
545 + 0].sampled_token_ids is None, "expect gpu tensor references to be None"
546
547 draft_worker.execute_model.assert_called_once_with(execute_model_req)
548 target_worker.execute_model.assert_called_once_with(execute_model_req)
@@ -556,7 +561,7 @@ def test_init_device(acceptance_sampler_method: str):
561 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
562
563 worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
559 - metrics_collector)
564 + False, metrics_collector)
565 worker.init_device()
566
567 draft_worker.init_device.assert_called_once()
@@ -707,6 +712,7 @@ def test_populate_seq_ids_with_bonus_tokens():
712 worker = SpecDecodeWorker(draft_worker,
713 target_worker,
714 mock_spec_decode_sampler("rejection_sampler"),
715 + disable_logprobs=False,
716 metrics_collector=metrics_collector)
717 # Initialize _seq_with_bonus_token_in_last_step with a set of sequence IDs.
718 # This set includes all sequence IDs in the batch as well as an additional
vllm/config.py
+17
@@ -894,6 +894,7 @@ class SpeculativeConfig:
894 draft_token_acceptance_method: str,
895 typical_acceptance_sampler_posterior_threshold: Optional[float],
896 typical_acceptance_sampler_posterior_alpha: Optional[float],
897 + disable_logprobs: Optional[bool],
898 ) -> Optional["SpeculativeConfig"]:
899 """Create a SpeculativeConfig if possible, else return None.
900
@@ -943,6 +944,11 @@ class SpeculativeConfig:
944 typical_acceptance_sampler_posterior_alpha (Optional[float]):
945 A scaling factor for the entropy-based threshold in the
946 TypicalAcceptanceSampler.
947 + disable_logprobs (Optional[bool]): If set to True, token log
948 + probabilities are not returned during speculative decoding.
949 + If set to False, token log probabilities are returned
950 + according to the log probability settings in SamplingParams.
951 + If not specified, it defaults to True.
952
953 Returns:
954 Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
@@ -1055,6 +1061,8 @@ class SpeculativeConfig:
1061 typical_acceptance_sampler_posterior_threshold = 0.09
1062 if typical_acceptance_sampler_posterior_alpha is None:
1063 typical_acceptance_sampler_posterior_alpha = 0.3
1064 + if disable_logprobs is None:
1065 + disable_logprobs = True
1066
1067 return SpeculativeConfig(
1068 draft_model_config,
@@ -1068,6 +1076,7 @@ class SpeculativeConfig:
1076 typical_acceptance_sampler_posterior_threshold,
1077 typical_acceptance_sampler_posterior_alpha=\
1078 typical_acceptance_sampler_posterior_alpha,
1079 + disable_logprobs=disable_logprobs
1080 )
1081
1082 @staticmethod
@@ -1152,6 +1161,7 @@ class SpeculativeConfig:
1161 draft_token_acceptance_method: str,
1162 typical_acceptance_sampler_posterior_threshold: float,
1163 typical_acceptance_sampler_posterior_alpha: float,
1164 + disable_logprobs: bool,
1165 ):
1166 """Create a SpeculativeConfig object.
1167
@@ -1178,6 +1188,12 @@ class SpeculativeConfig:
1188 typical_acceptance_sampler_posterior_alpha (Optional[float]):
1189 A scaling factor for the entropy-based threshold in the
1190 TypicalAcceptanceSampler.
1191 + disable_logprobs: If set to True, token log probabilities will not
1192 + be returned even if requested by sampling parameters. This
1193 + reduces latency by skipping logprob calculation in proposal
1194 + sampling, target sampling, and after accepted tokens are
1195 + determined. If set to False, log probabilities will be
1196 + returned.
1197 """
1198 self.draft_model_config = draft_model_config
1199 self.draft_parallel_config = draft_parallel_config
@@ -1191,6 +1207,7 @@ class SpeculativeConfig:
1207 typical_acceptance_sampler_posterior_threshold
1208 self.typical_acceptance_sampler_posterior_alpha = \
1209 typical_acceptance_sampler_posterior_alpha
1210 + self.disable_logprobs = disable_logprobs
1211
1212 self._verify_args()
1213
vllm/engine/arg_utils.py
+14
@@ -110,6 +110,7 @@ class EngineArgs:
110 typical_acceptance_sampler_posterior_threshold: Optional[float] = None
111 typical_acceptance_sampler_posterior_alpha: Optional[float] = None
112 qlora_adapter_name_or_path: Optional[str] = None
113 + disable_logprobs_during_spec_decoding: Optional[bool] = None
114
115 otlp_traces_endpoint: Optional[str] = None
116
@@ -592,6 +593,18 @@ class EngineArgs:
593 'to sqrt of --typical-acceptance-sampler-posterior-threshold '
594 'i.e. 0.3')
595
596 + parser.add_argument(
597 + '--disable-logprobs-during-spec-decoding',
598 + type=bool,
599 + default=EngineArgs.disable_logprobs_during_spec_decoding,
600 + help='If set to True, token log probabilities are not returned '
601 + 'during speculative decoding. If set to False, log probabilities '
602 + 'are returned according to the settings in SamplingParams. If '
603 + 'not specified, it defaults to True. Disabling log probabilities '
604 + 'during speculative decoding reduces latency by skipping logprob '
605 + 'calculation in proposal sampling, target sampling, and after '
606 + 'accepted tokens are determined.')
607 +
608 parser.add_argument('--model-loader-extra-config',
609 type=nullable_str,
610 default=EngineArgs.model_loader_extra_config,
@@ -736,6 +749,7 @@ class EngineArgs:
749 typical_acceptance_sampler_posterior_threshold,
750 typical_acceptance_sampler_posterior_alpha=self.
751 typical_acceptance_sampler_posterior_alpha,
752 + disable_logprobs=self.disable_logprobs_during_spec_decoding,
753 )
754
755 scheduler_config = SchedulerConfig(
vllm/spec_decode/spec_decode_worker.py
+185 -33
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.typical_acceptance_sampler import (
14 TypicalAcceptanceSampler)
15 from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
16 HiddenStates, SamplerOutput, SequenceGroupMetadata,
17 - get_all_seq_ids_and_request_ids)
17 + get_all_seq_ids, get_all_seq_ids_and_request_ids)
18 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
19 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
20 from vllm.spec_decode.interfaces import (SpeculativeProposals,
@@ -26,6 +26,7 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
26 from vllm.spec_decode.ngram_worker import NGramWorker
27 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
28 from vllm.spec_decode.smaller_tp_proposer_worker import SmallerTpProposerWorker
29 +from vllm.spec_decode.target_model_runner import TargetModelRunner
30 from vllm.spec_decode.util import (create_sequence_group_output,
31 get_all_num_logprobs,
32 get_sampled_token_logprobs, nvtx_range,
@@ -44,9 +45,15 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
45 speculative_config: SpeculativeConfig = kwargs.get("speculative_config")
46 assert speculative_config is not None
47
48 + draft_worker_kwargs = kwargs.copy()
49 +
50 + kwargs["model_runner_cls"] = TargetModelRunner
51 target_worker = Worker(*args, **kwargs)
52 + # Set the disable_logprobs variable in the TargetModelRunner instance
53 + # as per its value specified in the SpeculativeConfig.
54 + target_worker.model_runner.disable_logprobs =\
55 + speculative_config.disable_logprobs
56
49 - draft_worker_kwargs = kwargs.copy()
57 # Override draft-model specific worker args.
58 draft_worker_kwargs.update(
59 model_config=speculative_config.draft_model_config,
@@ -67,7 +74,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
74 typical_acceptance_sampler_posterior_threshold=speculative_config.
75 typical_acceptance_sampler_posterior_threshold,
76 typical_acceptance_sampler_posterior_alpha=speculative_config.
70 - typical_acceptance_sampler_posterior_alpha)
77 + typical_acceptance_sampler_posterior_alpha,
78 + disable_logprobs=speculative_config.disable_logprobs)
79
80 return spec_decode_worker
81
@@ -107,6 +115,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
115 draft_token_acceptance_method: str,
116 typical_acceptance_sampler_posterior_threshold: float,
117 typical_acceptance_sampler_posterior_alpha: float,
118 + disable_logprobs: bool,
119 ) -> "SpecDecodeWorker":
120
121 allow_zero_draft_token_step = True
@@ -161,6 +170,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
170 return SpecDecodeWorker(
171 proposer_worker,
172 scorer_worker,
173 + disable_logprobs=disable_logprobs,
174 disable_by_batch_size=disable_by_batch_size,
175 spec_decode_sampler=spec_decode_sampler,
176 allow_zero_draft_token_step=allow_zero_draft_token_step)
@@ -170,6 +180,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
180 proposer_worker: ProposerWorkerBase,
181 scorer_worker: WorkerBase,
182 spec_decode_sampler: SpecDecodeBaseSampler,
183 + disable_logprobs: bool,
184 metrics_collector: Optional[AsyncMetricsCollector] = None,
185 disable_by_batch_size: Optional[int] = None,
186 allow_zero_draft_token_step: Optional[bool] = True,
@@ -189,6 +200,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
200 types of sampler namely RejectionSampler and
201 TypicalAcceptanceSampler. 'spec_decode_sampler' is either an
202 instance of RejectionSampler or TypicalAcceptanceSampler.
203 + disable_logprobs: If set to True, token log probabilities will
204 + not be output in both the draft worker and the target worker.
205 + If set to False, log probabilities will be output by both.
206 disable_by_batch_size: If the batch size is larger than this,
207 disable speculative decoding for new incoming requests.
208 metrics_collector: Helper class for collecting metrics; can be set
@@ -222,6 +236,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
236 # Hidden states from target model to pass to proposer
237 # in the subsequent step.
238 self.previous_hidden_states: Optional[HiddenStates] = None
239 + self._disable_logprobs = disable_logprobs
240
241 def init_device(self) -> None:
242 """Initialize both scorer and proposer models.
@@ -357,7 +372,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
372 ) == 0 or disable_all_speculation:
373 return self._run_no_spec(execute_model_req,
374 skip_proposer=disable_all_speculation)
360 -
375 return self._run_speculative_decoding_step(execute_model_req,
376 num_lookahead_slots)
377
@@ -391,6 +405,42 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
405 # this state within spec decode worker.
406 seq_group_metadata.num_speculative_tokens = 0
407
408 + def _serialize_sampler_output_no_logprobs(
409 + self, execute_model_req: ExecuteModelRequest,
410 + sampler_output: SamplerOutput) -> SamplerOutput:
411 + """
412 + Creates and returns a `SamplerOutput` with only the sampled token IDs
413 + being serialized to CPU & populated in `CompletionSequenceGroupOutput`.
414 + All other parameters in `CompletionSequenceGroupOutput` related to log
415 + probabilities are skipped.
416 +
417 + Args:
418 + execute_model_req (ExecuteModelRequest): The model request that
419 + was executed.
420 + sampler_output (SamplerOutput): The output from the sampler with
421 + only GPU tensors populated.
422 +
423 + Returns:
424 + SamplerOutput: A new `SamplerOutput` instance containing a list of
425 + `CompletionSequenceGroupOutput` objects with only sampled token
426 + IDs populated.
427 + """
428 + seq_ids = get_all_seq_ids(execute_model_req.seq_group_metadata_list)
429 + sampled_token_ids_list = sampler_output.sampled_token_ids.tolist()
430 + completion_seq_group_output_list: List[
431 + CompletionSequenceGroupOutput] = []
432 + for index, seq_id in enumerate(seq_ids):
433 + completion_seq_group_output_list.append(
434 + create_sequence_group_output(
435 + token_id=sampled_token_ids_list[index][0],
436 + token_id_logprob_rank=-1,
437 + token_id_logprob=0.0,
438 + seq_id=seq_id,
439 + topk_token_ids=[],
440 + topk_logprobs=[],
441 + ))
442 + return SamplerOutput(outputs=completion_seq_group_output_list)
443 +
444 @nvtx_range("spec_decode_worker._run_no_spec")
445 def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
446 skip_proposer: bool) -> List[SamplerOutput]:
@@ -417,12 +467,17 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
467 self.previous_hidden_states.update(
468 execute_model_req.seq_group_metadata_list, hidden_states)
469
470 + sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
471 + execute_model_req=execute_model_req, sampler_output=sampler_output)
472 + if self._disable_logprobs else
473 + sampler_output)
474 +
475 # Clear device tensors from sampler output. This reduces communication
476 # overhead when the engine runs in a different process than the workers.
422 - sampler_output.probs = None
423 - sampler_output.sampled_tokens = None
477 + sampler_output.sampled_token_probs = None
478 + sampler_output.sampled_token_ids = None
479 sampler_output.logprobs = None
425 - return [sampler_output]
480 + return [sampler_output_to_return]
481
482 def _run_non_driver_rank(self) -> bool:
483 """Run proposer and verifier model in non-driver workers. This is used
@@ -480,7 +535,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
535 execute_model_req,
536 proposals,
537 )
483 -
538 accepted_token_ids, target_logprobs = self._verify_tokens(
539 execute_model_req.seq_group_metadata_list, proposal_scores,
540 proposals, execute_model_req.num_lookahead_slots)
@@ -601,25 +655,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
655 the same number of outputs.
656 """
657 batch_size, num_steps = accepted_token_ids.shape
604 -
605 - # Organize input tensors by step instead of by sequence.
606 - target_logprobs_by_step = target_logprobs.transpose(0, 1)
658 accepted_token_ids_by_step = accepted_token_ids.transpose(0, 1)
608 -
609 - # Get the logprobs/rank of the accepted tokens.
610 - (accepted_token_id_ranks_by_step,
611 - accepted_token_id_logprobs_by_step) = get_sampled_token_logprobs(
612 - logprob_tensor=target_logprobs_by_step,
613 - sampled_token_ids=accepted_token_ids_by_step,
614 - )
615 -
616 - # Get the top-k logprobs (which may or may not include the logprob of
617 - # the accepted token).
618 - (topk_logprobs_by_step,
619 - topk_indices_by_step) = target_logprobs_by_step.topk(
620 - k=self.scorer_worker.model_config.max_logprobs,
621 - dim=-1,
622 - )
659 + if self._disable_logprobs:
660 + # We are skipping the logprobs. Hence don't serialize the
661 + # logprobs related tensors from the GPU. Instead create
662 + # empty/dummy lists.
663 + (accepted_token_id_ranks_by_step,
664 + accepted_token_id_logprobs_by_step,
665 + topk_logprobs_by_step, topk_indices_by_step) =\
666 + self._create_dummy_logprob_lists(
667 + batch_size, num_steps,
668 + self.scorer_worker.model_config.max_logprobs)
669 + else:
670 + # Organize input tensors by step instead of by sequence.
671 + target_logprobs_by_step = target_logprobs.transpose(0, 1)
672 + # Serialize all tensors into Python lists.
673 + (accepted_token_id_ranks_by_step,
674 + accepted_token_id_logprobs_by_step,
675 + topk_logprobs_by_step, topk_indices_by_step) =\
676 + self._create_logprob_lists_from_tensors(
677 + target_logprobs_by_step, accepted_token_ids_by_step,
678 + self.scorer_worker.model_config.max_logprobs)
679
680 # Get the sequence ids and num_logprobs (sampling parameter) in the
681 # batch.
@@ -628,14 +684,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
684
685 num_logprobs_per_seq = get_all_num_logprobs(seq_group_metadata_list)
686
631 - # Serialize all tensors to CPU Python lists.
687 + # Serialize tensor to CPU Python list.
688 accepted_token_ids_by_step = accepted_token_ids_by_step.tolist()
633 - accepted_token_id_ranks_by_step = (
634 - accepted_token_id_ranks_by_step.tolist())
635 - accepted_token_id_logprobs_by_step = (
636 - accepted_token_id_logprobs_by_step.tolist())
637 - topk_logprobs_by_step = topk_logprobs_by_step.tolist()
638 - topk_indices_by_step = topk_indices_by_step.tolist()
689
690 # Construct the output on a per-step, per-sequence basis.
691 sampler_output_list: List[SamplerOutput] = []
@@ -677,6 +727,108 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
727 0].spec_decode_worker_metrics = maybe_rejsample_metrics
728 return sampler_output_list
729
730 + def _create_dummy_logprob_lists(
731 + self,
732 + batch_size: int,
733 + num_steps: int,
734 + num_top_k: int,
735 + ) -> Tuple[List[List[int]], List[List[float]],
736 + List[List[List[Optional[float]]]],
737 + List[List[List[Optional[int]]]]]:
738 + """
739 + Creates and returns four dummy lists representing token probabilities
740 + and their ranks.
741 +
742 + This method initializes and returns:
743 + - The ranks of the accepted tokens, shaped (num_steps, batch_size)
744 + - The log probabilities of the accepted tokens,
745 + shaped (num_steps, batch_size)
746 + - The log probabilities of the top k tokens,
747 + shaped (num_steps, batch_size, num_top_k)
748 + - The token IDs of the top k tokens,
749 + shaped (num_steps, batch_size, num_top_k)
750 +
751 + Args:
752 + batch_size (int): The size of the batch.
753 + num_steps (int): The number of steps in the sequence.
754 + num_top_k (int): The number of top-k token log probabilities to
755 + return.
756 +
757 + Returns:
758 + A tuple containing four dummy lists as described above.
759 + """
760 + accepted_token_id_ranks_by_step = [[-1] * batch_size
761 + for _ in range(num_steps)]
762 + accepted_token_id_logprobs_by_step = [[0.0] * batch_size
763 + for _ in range(num_steps)]
764 + topk_logprobs_by_step: List[List[List[Optional[float]]]] = [[
765 + [None] * num_top_k for _ in range(batch_size)
766 + ] for _ in range(num_steps)]
767 + topk_indices_by_step: List[List[List[Optional[int]]]] = [[
768 + [None] * num_top_k for _ in range(batch_size)
769 + ] for _ in range(num_steps)]
770 + return (accepted_token_id_ranks_by_step,
771 + accepted_token_id_logprobs_by_step, topk_logprobs_by_step,
772 + topk_indices_by_step)
773 +
774 + def _create_logprob_lists_from_tensors(
775 + self,
776 + target_logprobs_by_step: torch.Tensor,
777 + accepted_token_ids_by_step: torch.Tensor,
778 + num_top_k: int,
779 + ) -> Tuple[List[List[int]], List[List[float]],
780 + List[List[List[Optional[float]]]],
781 + List[List[List[Optional[int]]]]]:
782 + """
783 + Creates and returns four lists representing token probabilities and
784 + their ranks.
785 +
786 + This method initializes and returns four lists containing:
787 + - The ranks of the accepted tokens, shaped (num_steps, batch_size)
788 + - The log probabilities of the accepted tokens,
789 + shaped (num_steps, batch_size)
790 + - The log probabilities of the top k tokens,
791 + shaped (num_steps, batch_size, num_top_k)
792 + - The token IDs of the top k tokens,
793 + shaped (num_steps, batch_size, num_top_k)
794 +
795 + Args:
796 + target_logprobs_by_step (torch.Tensor): Tensor representing the
797 + log probabilities of the target model,
798 + shaped (num_steps, batch_size, vocab_size)
799 + accepted_token_ids_by_step (torch.Tensor): Tensor representing
800 + the accepted token_ids, shaped (num_steps, batch_size)
801 + num_top_k (int): The number of top-k token log probabilities to
802 + return.
803 +
804 + Returns:
805 + A tuple containing the lists as described above.
806 + """
807 + # Serialize all tensors to CPU Python lists.
808 + # Get the logprobs/rank of the accepted tokens.
809 + (accepted_token_id_ranks_by_step_tensor,
810 + accepted_token_id_logprobs_by_step_tensor
811 + ) = get_sampled_token_logprobs(
812 + logprob_tensor=target_logprobs_by_step,
813 + sampled_token_ids=accepted_token_ids_by_step,
814 + )
815 + # Get the top-k logprobs (which may or may not include the
816 + # logprob of the accepted token).
817 + (topk_logprobs_by_step_tensor,
818 + topk_indices_by_step_tensor) = target_logprobs_by_step.topk(
819 + k=num_top_k,
820 + dim=-1,
821 + )
822 + accepted_token_id_ranks_by_step = (
823 + accepted_token_id_ranks_by_step_tensor.tolist())
824 + accepted_token_id_logprobs_by_step = (
825 + accepted_token_id_logprobs_by_step_tensor.tolist())
826 + topk_logprobs_by_step = topk_logprobs_by_step_tensor.tolist()
827 + topk_indices_by_step = topk_indices_by_step_tensor.tolist()
828 + return (accepted_token_id_ranks_by_step,
829 + accepted_token_id_logprobs_by_step, topk_logprobs_by_step,
830 + topk_indices_by_step)
831 +
832 def _track_finished_requests(self, execute_model_req: ExecuteModelRequest):
833 """
834 Removes the finished requests and their associated sequence ids from
vllm/spec_decode/target_model_runner.py new
+69
@@ -0,0 +1,69 @@
1 +from typing import List, Optional
2 +
3 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
4 + ModelConfig, MultiModalConfig, ParallelConfig,
5 + PromptAdapterConfig, SchedulerConfig)
6 +from vllm.sequence import SequenceGroupMetadata
7 +from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
8 + ModelRunner)
9 +
10 +
11 +class TargetModelRunner(ModelRunner):
12 + """Specialized model runner for speculative decoding target model.
13 + In speculative decoding, the log probabilities selected finally may not
14 + be the same ones as selected by the target model sampling. This means
15 + that the time spent in the log probability calculation of the target model
16 + is time wasted, since we calculate log probabilities after deciding which
17 + tokens are accepted. For this reason disabling log probabilities in the
18 + target model will make decode faster. The model runner sets the
19 + SamplingMetadata parameters according to whether log probabilities are
20 + requested or not.
21 + """
22 +
23 + def __init__(self,
24 + model_config: ModelConfig,
25 + parallel_config: ParallelConfig,
26 + scheduler_config: SchedulerConfig,
27 + device_config: DeviceConfig,
28 + cache_config: CacheConfig,
29 + load_config: LoadConfig,
30 + lora_config: Optional[LoRAConfig],
31 + kv_cache_dtype: Optional[str] = "auto",
32 + is_driver_worker: bool = False,
33 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
34 + multimodal_config: Optional[MultiModalConfig] = None,
35 + return_hidden_states: bool = False):
36 + # An internal boolean member variable to indicate if token log
37 + # probabilities are needed or not.
38 + self.disable_logprobs = True
39 + super().__init__(
40 + model_config=model_config,
41 + parallel_config=parallel_config,
42 + scheduler_config=scheduler_config,
43 + device_config=device_config,
44 + cache_config=cache_config,
45 + load_config=load_config,
46 + lora_config=lora_config,
47 + kv_cache_dtype=kv_cache_dtype,
48 + is_driver_worker=is_driver_worker,
49 + multimodal_config=multimodal_config,
50 + prompt_adapter_config=prompt_adapter_config,
51 + return_hidden_states=return_hidden_states,
52 + )
53 +
54 + def prepare_model_input(
55 + self,
56 + seq_group_metadata_list: List[SequenceGroupMetadata],
57 + virtual_engine: int = 0,
58 + finished_requests_ids: Optional[List[str]] = None
59 + ) -> ModelInputForGPUWithSamplingMetadata:
60 + model_input: ModelInputForGPUWithSamplingMetadata = super(
61 + ).prepare_model_input(seq_group_metadata_list, virtual_engine,
62 + finished_requests_ids)
63 + # If token log probabilities is disabled then skip generating sampler
64 + # CPU output. We directly serialize the GPU sampled_token_id tensors
65 + # as needed. If log probabilities is enabled then synchronize all the
66 + # sampling related tensors which includes the logprobs tensors.
67 + model_input.sampling_metadata.skip_sampler_cpu_output = (
68 + self.disable_logprobs)
69 + return model_input
vllm/spec_decode/util.py
+4 -4
@@ -1,5 +1,5 @@
1 from contextlib import contextmanager
2 -from typing import Dict, List, Tuple
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5
@@ -53,8 +53,8 @@ def create_sequence_group_output(
53 token_id_logprob_rank: int,
54 token_id_logprob: float,
55 seq_id: SeqId,
56 - topk_token_ids: List[int],
57 - topk_logprobs: List[float],
56 + topk_token_ids: List[Optional[int]],
57 + topk_logprobs: List[Optional[float]],
58 ) -> CompletionSequenceGroupOutput:
59 """Create a SequenceGroupOutput given the sampling results.
60
@@ -68,7 +68,7 @@ def create_sequence_group_output(
68 """
69 # vLLM logprobs always include the sampled token. In addition, the user may
70 # request topk-logprobs (where top-k varies per user up to max_logprobs).
71 - logprobs: Dict[int, Logprob] = {
71 + logprobs: Dict[Optional[int], Logprob] = {
72 token_id: Logprob(
73 logprob=token_id_logprob,
74 rank=token_id_logprob_rank,