A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Spec Decode] Disable Log Prob serialization to CPU for spec decoding for both draft and target models. (#6485)
sroy745 committed
Jul 20, 2024 at 23:58 UTC
14f91fe67c2342f2fe859dc6a5c40810df0e1c61
8 files changed
+332
-63
tests/spec_decode/e2e/test_logprobs.py
+29
-19
@@ -22,10 +22,12 @@ from .conftest import get_logprobs_from_llm_generator
22
}])
23
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
24
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
25
-@pytest.mark.parametrize("test_llm_kwargs", [{
26
- "speculative_model": "JackFram/llama-160m",
27
- "num_speculative_tokens": 3,
28
-}])
25
+@pytest.mark.parametrize("test_llm_kwargs",
26
+ [{
27
+ "speculative_model": "JackFram/llama-160m",
28
+ "num_speculative_tokens": 3,
29
+ "disable_logprobs_during_spec_decoding": False,
30
+ }])
31
@pytest.mark.parametrize("batch_size", [8])
32
@pytest.mark.parametrize(
33
"output_len",
@@ -59,10 +61,12 @@ def test_logprobs_equality(baseline_llm_generator, test_llm_generator,
61
}])
62
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
63
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
62
-@pytest.mark.parametrize("test_llm_kwargs", [{
63
- "speculative_model": "JackFram/llama-160m",
64
- "num_speculative_tokens": 3,
65
-}])
64
+@pytest.mark.parametrize("test_llm_kwargs",
65
+ [{
66
+ "speculative_model": "JackFram/llama-160m",
67
+ "num_speculative_tokens": 3,
68
+ "disable_logprobs_during_spec_decoding": False,
69
+ }])
70
@pytest.mark.parametrize("batch_size", [1])
71
@pytest.mark.parametrize("num_logprobs", [6])
72
@pytest.mark.parametrize(
@@ -99,13 +103,16 @@ def test_diff_num_logprobs(baseline_llm_generator, test_llm_generator,
103
}])
104
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
105
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
102
-@pytest.mark.parametrize("test_llm_kwargs", [{
103
- "speculative_model": "JackFram/llama-160m",
104
- "num_speculative_tokens": 3,
105
-}, {
106
- "speculative_model": "JackFram/llama-160m",
107
- "num_speculative_tokens": 6,
108
-}])
106
+@pytest.mark.parametrize("test_llm_kwargs",
107
+ [{
108
+ "speculative_model": "JackFram/llama-160m",
109
+ "num_speculative_tokens": 3,
110
+ "disable_logprobs_during_spec_decoding": False,
111
+ }, {
112
+ "speculative_model": "JackFram/llama-160m",
113
+ "num_speculative_tokens": 6,
114
+ "disable_logprobs_during_spec_decoding": False,
115
+ }])
116
@pytest.mark.parametrize("batch_size", [8])
117
@pytest.mark.parametrize(
118
"output_len",
@@ -143,6 +150,7 @@ def test_logprobs_different_k(baseline_llm_generator, test_llm_generator,
150
[{
151
"speculative_model": "JackFram/llama-160m",
152
"num_speculative_tokens": 3,
153
+ "disable_logprobs_during_spec_decoding": False,
154
155
# Artificially limit the draft model max model len; this forces vLLM
156
# to skip speculation once the sequences grow beyond 32-k tokens.
@@ -181,10 +189,12 @@ def test_logprobs_when_skip_speculation(baseline_llm_generator,
189
}])
190
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
191
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
184
-@pytest.mark.parametrize("test_llm_kwargs", [{
185
- "speculative_model": "JackFram/llama-160m",
186
- "num_speculative_tokens": 3,
187
-}])
192
+@pytest.mark.parametrize("test_llm_kwargs",
193
+ [{
194
+ "speculative_model": "JackFram/llama-160m",
195
+ "num_speculative_tokens": 3,
196
+ "disable_logprobs_during_spec_decoding": False,
197
+ }])
198
@pytest.mark.parametrize("batch_size", [1])
199
@pytest.mark.parametrize(
200
"output_len",
tests/spec_decode/test_dynamic_spec_decode.py
+1
@@ -32,6 +32,7 @@ def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int,
32
scorer_worker=target_worker,
33
spec_decode_sampler=mock_spec_decode_sampler(
34
acceptance_sampler_method),
35
+ disable_logprobs=False,
36
metrics_collector=metrics_collector,
37
disable_by_batch_size=disable_by_batch_size)
38
tests/spec_decode/test_spec_decode_worker.py
+13
-7
@@ -381,6 +381,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool,
381
worker = SpecDecodeWorker(draft_worker,
382
target_worker,
383
spec_decode_sampler,
384
+ disable_logprobs=False,
385
metrics_collector=metrics_collector)
386
worker.init_device()
387
@@ -479,7 +480,8 @@ def test_k_equals_zero(k: int, batch_size: int,
480
481
worker = SpecDecodeWorker(
482
draft_worker, target_worker,
482
- mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
483
+ mock_spec_decode_sampler(acceptance_sampler_method), False,
484
+ metrics_collector)
485
486
seq_group_metadata_list, _, _ = create_batch(batch_size,
487
k,
@@ -490,9 +492,10 @@ def test_k_equals_zero(k: int, batch_size: int,
492
out = worker.execute_model(execute_model_req=execute_model_req)
493
494
assert len(out) == 1, f"expected only one token output when {k=}"
493
- assert out[0].probs is None, "expect gpu tensor references to be None"
495
+ assert out[0].sampled_token_probs is None, (
496
+ "expect gpu tensor references to be None")
497
assert out[
495
- 0].sampled_tokens is None, "expect gpu tensor references to be None"
498
+ 0].sampled_token_ids is None, "expect gpu tensor references to be None"
499
500
draft_worker.execute_model.assert_called_once_with(execute_model_req)
501
target_worker.execute_model.assert_called_once_with(execute_model_req)
@@ -524,7 +527,8 @@ def test_empty_input_batch(k: int, batch_size: int,
527
528
worker = SpecDecodeWorker(
529
draft_worker, target_worker,
527
- mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
530
+ mock_spec_decode_sampler(acceptance_sampler_method), False,
531
+ metrics_collector)
532
533
seq_group_metadata_list, _, _ = create_batch(batch_size,
534
k,
@@ -535,9 +539,10 @@ def test_empty_input_batch(k: int, batch_size: int,
539
out = worker.execute_model(execute_model_req=execute_model_req)
540
541
assert len(out) == 1, f"expected only one token output when {k=}"
538
- assert out[0].probs is None, "expect gpu tensor references to be None"
542
+ assert out[0].sampled_token_probs is None, (
543
+ "expect gpu tensor references to be None")
544
assert out[
540
- 0].sampled_tokens is None, "expect gpu tensor references to be None"
545
+ 0].sampled_token_ids is None, "expect gpu tensor references to be None"
546
547
draft_worker.execute_model.assert_called_once_with(execute_model_req)
548
target_worker.execute_model.assert_called_once_with(execute_model_req)
@@ -556,7 +561,7 @@ def test_init_device(acceptance_sampler_method: str):
561
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
562
563
worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
559
- metrics_collector)
564
+ False, metrics_collector)
565
worker.init_device()
566
567
draft_worker.init_device.assert_called_once()
@@ -707,6 +712,7 @@ def test_populate_seq_ids_with_bonus_tokens():
712
worker = SpecDecodeWorker(draft_worker,
713
target_worker,
714
mock_spec_decode_sampler("rejection_sampler"),
715
+ disable_logprobs=False,
716
metrics_collector=metrics_collector)
717
# Initialize _seq_with_bonus_token_in_last_step with a set of sequence IDs.
718
# This set includes all sequence IDs in the batch as well as an additional
vllm/config.py
+17
@@ -894,6 +894,7 @@ class SpeculativeConfig:
894
draft_token_acceptance_method: str,
895
typical_acceptance_sampler_posterior_threshold: Optional[float],
896
typical_acceptance_sampler_posterior_alpha: Optional[float],
897
+ disable_logprobs: Optional[bool],
898
) -> Optional["SpeculativeConfig"]:
899
"""Create a SpeculativeConfig if possible, else return None.
900
@@ -943,6 +944,11 @@ class SpeculativeConfig:
944
typical_acceptance_sampler_posterior_alpha (Optional[float]):
945
A scaling factor for the entropy-based threshold in the
946
TypicalAcceptanceSampler.
947
+ disable_logprobs (Optional[bool]): If set to True, token log
948
+ probabilities are not returned during speculative decoding.
949
+ If set to False, token log probabilities are returned
950
+ according to the log probability settings in SamplingParams.
951
+ If not specified, it defaults to True.
952
953
Returns:
954
Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
@@ -1055,6 +1061,8 @@ class SpeculativeConfig:
1061
typical_acceptance_sampler_posterior_threshold = 0.09
1062
if typical_acceptance_sampler_posterior_alpha is None:
1063
typical_acceptance_sampler_posterior_alpha = 0.3
1064
+ if disable_logprobs is None:
1065
+ disable_logprobs = True
1066
1067
return SpeculativeConfig(
1068
draft_model_config,
@@ -1068,6 +1076,7 @@ class SpeculativeConfig:
1076
typical_acceptance_sampler_posterior_threshold,
1077
typical_acceptance_sampler_posterior_alpha=\
1078
typical_acceptance_sampler_posterior_alpha,
1079
+ disable_logprobs=disable_logprobs
1080
)
1081
1082
@staticmethod
@@ -1152,6 +1161,7 @@ class SpeculativeConfig:
1161
draft_token_acceptance_method: str,
1162
typical_acceptance_sampler_posterior_threshold: float,
1163
typical_acceptance_sampler_posterior_alpha: float,
1164
+ disable_logprobs: bool,
1165
):
1166
"""Create a SpeculativeConfig object.
1167
@@ -1178,6 +1188,12 @@ class SpeculativeConfig:
1188
typical_acceptance_sampler_posterior_alpha (Optional[float]):
1189
A scaling factor for the entropy-based threshold in the
1190
TypicalAcceptanceSampler.
1191
+ disable_logprobs: If set to True, token log probabilities will not
1192
+ be returned even if requested by sampling parameters. This
1193
+ reduces latency by skipping logprob calculation in proposal
1194
+ sampling, target sampling, and after accepted tokens are
1195
+ determined. If set to False, log probabilities will be
1196
+ returned.
1197
"""
1198
self.draft_model_config = draft_model_config
1199
self.draft_parallel_config = draft_parallel_config
@@ -1191,6 +1207,7 @@ class SpeculativeConfig:
1207
typical_acceptance_sampler_posterior_threshold
1208
self.typical_acceptance_sampler_posterior_alpha = \
1209
typical_acceptance_sampler_posterior_alpha
1210
+ self.disable_logprobs = disable_logprobs
1211
1212
self._verify_args()
1213
vllm/engine/arg_utils.py
+14
@@ -110,6 +110,7 @@ class EngineArgs:
110
typical_acceptance_sampler_posterior_threshold: Optional[float] = None
111
typical_acceptance_sampler_posterior_alpha: Optional[float] = None
112
qlora_adapter_name_or_path: Optional[str] = None
113
+ disable_logprobs_during_spec_decoding: Optional[bool] = None
114
115
otlp_traces_endpoint: Optional[str] = None
116
@@ -592,6 +593,18 @@ class EngineArgs:
593
'to sqrt of --typical-acceptance-sampler-posterior-threshold '
594
'i.e. 0.3')
595
596
+ parser.add_argument(
597
+ '--disable-logprobs-during-spec-decoding',
598
+ type=bool,
599
+ default=EngineArgs.disable_logprobs_during_spec_decoding,
600
+ help='If set to True, token log probabilities are not returned '
601
+ 'during speculative decoding. If set to False, log probabilities '
602
+ 'are returned according to the settings in SamplingParams. If '
603
+ 'not specified, it defaults to True. Disabling log probabilities '
604
+ 'during speculative decoding reduces latency by skipping logprob '
605
+ 'calculation in proposal sampling, target sampling, and after '
606
+ 'accepted tokens are determined.')
607
+
608
parser.add_argument('--model-loader-extra-config',
609
type=nullable_str,
610
default=EngineArgs.model_loader_extra_config,
@@ -736,6 +749,7 @@ class EngineArgs:
749
typical_acceptance_sampler_posterior_threshold,
750
typical_acceptance_sampler_posterior_alpha=self.
751
typical_acceptance_sampler_posterior_alpha,
752
+ disable_logprobs=self.disable_logprobs_during_spec_decoding,
753
)
754
755
scheduler_config = SchedulerConfig(
vllm/spec_decode/spec_decode_worker.py
+185
-33
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.typical_acceptance_sampler import (
14
TypicalAcceptanceSampler)
15
from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
16
HiddenStates, SamplerOutput, SequenceGroupMetadata,
17
- get_all_seq_ids_and_request_ids)
17
+ get_all_seq_ids, get_all_seq_ids_and_request_ids)
18
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
19
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
20
from vllm.spec_decode.interfaces import (SpeculativeProposals,
@@ -26,6 +26,7 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
26
from vllm.spec_decode.ngram_worker import NGramWorker
27
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
28
from vllm.spec_decode.smaller_tp_proposer_worker import SmallerTpProposerWorker
29
+from vllm.spec_decode.target_model_runner import TargetModelRunner
30
from vllm.spec_decode.util import (create_sequence_group_output,
31
get_all_num_logprobs,
32
get_sampled_token_logprobs, nvtx_range,
@@ -44,9 +45,15 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
45
speculative_config: SpeculativeConfig = kwargs.get("speculative_config")
46
assert speculative_config is not None
47
48
+ draft_worker_kwargs = kwargs.copy()
49
+
50
+ kwargs["model_runner_cls"] = TargetModelRunner
51
target_worker = Worker(*args, **kwargs)
52
+ # Set the disable_logprobs variable in the TargetModelRunner instance
53
+ # as per its value specified in the SpeculativeConfig.
54
+ target_worker.model_runner.disable_logprobs =\
55
+ speculative_config.disable_logprobs
56
49
- draft_worker_kwargs = kwargs.copy()
57
# Override draft-model specific worker args.
58
draft_worker_kwargs.update(
59
model_config=speculative_config.draft_model_config,
@@ -67,7 +74,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
74
typical_acceptance_sampler_posterior_threshold=speculative_config.
75
typical_acceptance_sampler_posterior_threshold,
76
typical_acceptance_sampler_posterior_alpha=speculative_config.
70
- typical_acceptance_sampler_posterior_alpha)
77
+ typical_acceptance_sampler_posterior_alpha,
78
+ disable_logprobs=speculative_config.disable_logprobs)
79
80
return spec_decode_worker
81
@@ -107,6 +115,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
115
draft_token_acceptance_method: str,
116
typical_acceptance_sampler_posterior_threshold: float,
117
typical_acceptance_sampler_posterior_alpha: float,
118
+ disable_logprobs: bool,
119
) -> "SpecDecodeWorker":
120
121
allow_zero_draft_token_step = True
@@ -161,6 +170,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
170
return SpecDecodeWorker(
171
proposer_worker,
172
scorer_worker,
173
+ disable_logprobs=disable_logprobs,
174
disable_by_batch_size=disable_by_batch_size,
175
spec_decode_sampler=spec_decode_sampler,
176
allow_zero_draft_token_step=allow_zero_draft_token_step)
@@ -170,6 +180,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
180
proposer_worker: ProposerWorkerBase,
181
scorer_worker: WorkerBase,
182
spec_decode_sampler: SpecDecodeBaseSampler,
183
+ disable_logprobs: bool,
184
metrics_collector: Optional[AsyncMetricsCollector] = None,
185
disable_by_batch_size: Optional[int] = None,
186
allow_zero_draft_token_step: Optional[bool] = True,
@@ -189,6 +200,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
200
types of sampler namely RejectionSampler and
201
TypicalAcceptanceSampler. 'spec_decode_sampler' is either an
202
instance of RejectionSampler or TypicalAcceptanceSampler.
203
+ disable_logprobs: If set to True, token log probabilities will
204
+ not be output in both the draft worker and the target worker.
205
+ If set to False, log probabilities will be output by both.
206
disable_by_batch_size: If the batch size is larger than this,
207
disable speculative decoding for new incoming requests.
208
metrics_collector: Helper class for collecting metrics; can be set
@@ -222,6 +236,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
236
# Hidden states from target model to pass to proposer
237
# in the subsequent step.
238
self.previous_hidden_states: Optional[HiddenStates] = None
239
+ self._disable_logprobs = disable_logprobs
240
241
def init_device(self) -> None:
242
"""Initialize both scorer and proposer models.
@@ -357,7 +372,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
372
) == 0 or disable_all_speculation:
373
return self._run_no_spec(execute_model_req,
374
skip_proposer=disable_all_speculation)
360
-
375
return self._run_speculative_decoding_step(execute_model_req,
376
num_lookahead_slots)
377
@@ -391,6 +405,42 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
405
# this state within spec decode worker.
406
seq_group_metadata.num_speculative_tokens = 0
407
408
+ def _serialize_sampler_output_no_logprobs(
409
+ self, execute_model_req: ExecuteModelRequest,
410
+ sampler_output: SamplerOutput) -> SamplerOutput:
411
+ """
412
+ Creates and returns a `SamplerOutput` with only the sampled token IDs
413
+ being serialized to CPU & populated in `CompletionSequenceGroupOutput`.
414
+ All other parameters in `CompletionSequenceGroupOutput` related to log
415
+ probabilities are skipped.
416
+
417
+ Args:
418
+ execute_model_req (ExecuteModelRequest): The model request that
419
+ was executed.
420
+ sampler_output (SamplerOutput): The output from the sampler with
421
+ only GPU tensors populated.
422
+
423
+ Returns:
424
+ SamplerOutput: A new `SamplerOutput` instance containing a list of
425
+ `CompletionSequenceGroupOutput` objects with only sampled token
426
+ IDs populated.
427
+ """
428
+ seq_ids = get_all_seq_ids(execute_model_req.seq_group_metadata_list)
429
+ sampled_token_ids_list = sampler_output.sampled_token_ids.tolist()
430
+ completion_seq_group_output_list: List[
431
+ CompletionSequenceGroupOutput] = []
432
+ for index, seq_id in enumerate(seq_ids):
433
+ completion_seq_group_output_list.append(
434
+ create_sequence_group_output(
435
+ token_id=sampled_token_ids_list[index][0],
436
+ token_id_logprob_rank=-1,
437
+ token_id_logprob=0.0,
438
+ seq_id=seq_id,
439
+ topk_token_ids=[],
440
+ topk_logprobs=[],
441
+ ))
442
+ return SamplerOutput(outputs=completion_seq_group_output_list)
443
+
444
@nvtx_range("spec_decode_worker._run_no_spec")
445
def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
446
skip_proposer: bool) -> List[SamplerOutput]:
@@ -417,12 +467,17 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
467
self.previous_hidden_states.update(
468
execute_model_req.seq_group_metadata_list, hidden_states)
469
470
+ sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
471
+ execute_model_req=execute_model_req, sampler_output=sampler_output)
472
+ if self._disable_logprobs else
473
+ sampler_output)
474
+
475
# Clear device tensors from sampler output. This reduces communication
476
# overhead when the engine runs in a different process than the workers.
422
- sampler_output.probs = None
423
- sampler_output.sampled_tokens = None
477
+ sampler_output.sampled_token_probs = None
478
+ sampler_output.sampled_token_ids = None
479
sampler_output.logprobs = None
425
- return [sampler_output]
480
+ return [sampler_output_to_return]
481
482
def _run_non_driver_rank(self) -> bool:
483
"""Run proposer and verifier model in non-driver workers. This is used
@@ -480,7 +535,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
535
execute_model_req,
536
proposals,
537
)
483
-
538
accepted_token_ids, target_logprobs = self._verify_tokens(
539
execute_model_req.seq_group_metadata_list, proposal_scores,
540
proposals, execute_model_req.num_lookahead_slots)
@@ -601,25 +655,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
655
the same number of outputs.
656
"""
657
batch_size, num_steps = accepted_token_ids.shape
604
-
605
- # Organize input tensors by step instead of by sequence.
606
- target_logprobs_by_step = target_logprobs.transpose(0, 1)
658
accepted_token_ids_by_step = accepted_token_ids.transpose(0, 1)
608
-
609
- # Get the logprobs/rank of the accepted tokens.
610
- (accepted_token_id_ranks_by_step,
611
- accepted_token_id_logprobs_by_step) = get_sampled_token_logprobs(
612
- logprob_tensor=target_logprobs_by_step,
613
- sampled_token_ids=accepted_token_ids_by_step,
614
- )
615
-
616
- # Get the top-k logprobs (which may or may not include the logprob of
617
- # the accepted token).
618
- (topk_logprobs_by_step,
619
- topk_indices_by_step) = target_logprobs_by_step.topk(
620
- k=self.scorer_worker.model_config.max_logprobs,
621
- dim=-1,
622
- )
659
+ if self._disable_logprobs:
660
+ # We are skipping the logprobs. Hence don't serialize the
661
+ # logprobs related tensors from the GPU. Instead create
662
+ # empty/dummy lists.
663
+ (accepted_token_id_ranks_by_step,
664
+ accepted_token_id_logprobs_by_step,
665
+ topk_logprobs_by_step, topk_indices_by_step) =\
666
+ self._create_dummy_logprob_lists(
667
+ batch_size, num_steps,
668
+ self.scorer_worker.model_config.max_logprobs)
669
+ else:
670
+ # Organize input tensors by step instead of by sequence.
671
+ target_logprobs_by_step = target_logprobs.transpose(0, 1)
672
+ # Serialize all tensors into Python lists.
673
+ (accepted_token_id_ranks_by_step,
674
+ accepted_token_id_logprobs_by_step,
675
+ topk_logprobs_by_step, topk_indices_by_step) =\
676
+ self._create_logprob_lists_from_tensors(
677
+ target_logprobs_by_step, accepted_token_ids_by_step,
678
+ self.scorer_worker.model_config.max_logprobs)
679
680
# Get the sequence ids and num_logprobs (sampling parameter) in the
681
# batch.
@@ -628,14 +684,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
684
685
num_logprobs_per_seq = get_all_num_logprobs(seq_group_metadata_list)
686
631
- # Serialize all tensors to CPU Python lists.
687
+ # Serialize tensor to CPU Python list.
688
accepted_token_ids_by_step = accepted_token_ids_by_step.tolist()
633
- accepted_token_id_ranks_by_step = (
634
- accepted_token_id_ranks_by_step.tolist())
635
- accepted_token_id_logprobs_by_step = (
636
- accepted_token_id_logprobs_by_step.tolist())
637
- topk_logprobs_by_step = topk_logprobs_by_step.tolist()
638
- topk_indices_by_step = topk_indices_by_step.tolist()
689
690
# Construct the output on a per-step, per-sequence basis.
691
sampler_output_list: List[SamplerOutput] = []
@@ -677,6 +727,108 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
727
0].spec_decode_worker_metrics = maybe_rejsample_metrics
728
return sampler_output_list
729
730
+ def _create_dummy_logprob_lists(
731
+ self,
732
+ batch_size: int,
733
+ num_steps: int,
734
+ num_top_k: int,
735
+ ) -> Tuple[List[List[int]], List[List[float]],
736
+ List[List[List[Optional[float]]]],
737
+ List[List[List[Optional[int]]]]]:
738
+ """
739
+ Creates and returns four dummy lists representing token probabilities
740
+ and their ranks.
741
+
742
+ This method initializes and returns:
743
+ - The ranks of the accepted tokens, shaped (num_steps, batch_size)
744
+ - The log probabilities of the accepted tokens,
745
+ shaped (num_steps, batch_size)
746
+ - The log probabilities of the top k tokens,
747
+ shaped (num_steps, batch_size, num_top_k)
748
+ - The token IDs of the top k tokens,
749
+ shaped (num_steps, batch_size, num_top_k)
750
+
751
+ Args:
752
+ batch_size (int): The size of the batch.
753
+ num_steps (int): The number of steps in the sequence.
754
+ num_top_k (int): The number of top-k token log probabilities to
755
+ return.
756
+
757
+ Returns:
758
+ A tuple containing four dummy lists as described above.
759
+ """
760
+ accepted_token_id_ranks_by_step = [[-1] * batch_size
761
+ for _ in range(num_steps)]
762
+ accepted_token_id_logprobs_by_step = [[0.0] * batch_size
763
+ for _ in range(num_steps)]
764
+ topk_logprobs_by_step: List[List[List[Optional[float]]]] = [[
765
+ [None] * num_top_k for _ in range(batch_size)
766
+ ] for _ in range(num_steps)]
767
+ topk_indices_by_step: List[List[List[Optional[int]]]] = [[
768
+ [None] * num_top_k for _ in range(batch_size)
769
+ ] for _ in range(num_steps)]
770
+ return (accepted_token_id_ranks_by_step,
771
+ accepted_token_id_logprobs_by_step, topk_logprobs_by_step,
772
+ topk_indices_by_step)
773
+
774
+ def _create_logprob_lists_from_tensors(
775
+ self,
776
+ target_logprobs_by_step: torch.Tensor,
777
+ accepted_token_ids_by_step: torch.Tensor,
778
+ num_top_k: int,
779
+ ) -> Tuple[List[List[int]], List[List[float]],
780
+ List[List[List[Optional[float]]]],
781
+ List[List[List[Optional[int]]]]]:
782
+ """
783
+ Creates and returns four lists representing token probabilities and
784
+ their ranks.
785
+
786
+ This method initializes and returns four lists containing:
787
+ - The ranks of the accepted tokens, shaped (num_steps, batch_size)
788
+ - The log probabilities of the accepted tokens,
789
+ shaped (num_steps, batch_size)
790
+ - The log probabilities of the top k tokens,
791
+ shaped (num_steps, batch_size, num_top_k)
792
+ - The token IDs of the top k tokens,
793
+ shaped (num_steps, batch_size, num_top_k)
794
+
795
+ Args:
796
+ target_logprobs_by_step (torch.Tensor): Tensor representing the
797
+ log probabilities of the target model,
798
+ shaped (num_steps, batch_size, vocab_size)
799
+ accepted_token_ids_by_step (torch.Tensor): Tensor representing
800
+ the accepted token_ids, shaped (num_steps, batch_size)
801
+ num_top_k (int): The number of top-k token log probabilities to
802
+ return.
803
+
804
+ Returns:
805
+ A tuple containing the lists as described above.
806
+ """
807
+ # Serialize all tensors to CPU Python lists.
808
+ # Get the logprobs/rank of the accepted tokens.
809
+ (accepted_token_id_ranks_by_step_tensor,
810
+ accepted_token_id_logprobs_by_step_tensor
811
+ ) = get_sampled_token_logprobs(
812
+ logprob_tensor=target_logprobs_by_step,
813
+ sampled_token_ids=accepted_token_ids_by_step,
814
+ )
815
+ # Get the top-k logprobs (which may or may not include the
816
+ # logprob of the accepted token).
817
+ (topk_logprobs_by_step_tensor,
818
+ topk_indices_by_step_tensor) = target_logprobs_by_step.topk(
819
+ k=num_top_k,
820
+ dim=-1,
821
+ )
822
+ accepted_token_id_ranks_by_step = (
823
+ accepted_token_id_ranks_by_step_tensor.tolist())
824
+ accepted_token_id_logprobs_by_step = (
825
+ accepted_token_id_logprobs_by_step_tensor.tolist())
826
+ topk_logprobs_by_step = topk_logprobs_by_step_tensor.tolist()
827
+ topk_indices_by_step = topk_indices_by_step_tensor.tolist()
828
+ return (accepted_token_id_ranks_by_step,
829
+ accepted_token_id_logprobs_by_step, topk_logprobs_by_step,
830
+ topk_indices_by_step)
831
+
832
def _track_finished_requests(self, execute_model_req: ExecuteModelRequest):
833
"""
834
Removes the finished requests and their associated sequence ids from
vllm/spec_decode/target_model_runner.py
new
+69
@@ -0,0 +1,69 @@
1
+from typing import List, Optional
2
+
3
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
4
+ ModelConfig, MultiModalConfig, ParallelConfig,
5
+ PromptAdapterConfig, SchedulerConfig)
6
+from vllm.sequence import SequenceGroupMetadata
7
+from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
8
+ ModelRunner)
9
+
10
+
11
+class TargetModelRunner(ModelRunner):
12
+ """Specialized model runner for speculative decoding target model.
13
+ In speculative decoding, the log probabilities selected finally may not
14
+ be the same ones as selected by the target model sampling. This means
15
+ that the time spent in the log probability calculation of the target model
16
+ is time wasted, since we calculate log probabilities after deciding which
17
+ tokens are accepted. For this reason disabling log probabilities in the
18
+ target model will make decode faster. The model runner sets the
19
+ SamplingMetadata parameters according to whether log probabilities are
20
+ requested or not.
21
+ """
22
+
23
+ def __init__(self,
24
+ model_config: ModelConfig,
25
+ parallel_config: ParallelConfig,
26
+ scheduler_config: SchedulerConfig,
27
+ device_config: DeviceConfig,
28
+ cache_config: CacheConfig,
29
+ load_config: LoadConfig,
30
+ lora_config: Optional[LoRAConfig],
31
+ kv_cache_dtype: Optional[str] = "auto",
32
+ is_driver_worker: bool = False,
33
+ prompt_adapter_config: Optional[PromptAdapterConfig] = None,
34
+ multimodal_config: Optional[MultiModalConfig] = None,
35
+ return_hidden_states: bool = False):
36
+ # An internal boolean member variable to indicate if token log
37
+ # probabilities are needed or not.
38
+ self.disable_logprobs = True
39
+ super().__init__(
40
+ model_config=model_config,
41
+ parallel_config=parallel_config,
42
+ scheduler_config=scheduler_config,
43
+ device_config=device_config,
44
+ cache_config=cache_config,
45
+ load_config=load_config,
46
+ lora_config=lora_config,
47
+ kv_cache_dtype=kv_cache_dtype,
48
+ is_driver_worker=is_driver_worker,
49
+ multimodal_config=multimodal_config,
50
+ prompt_adapter_config=prompt_adapter_config,
51
+ return_hidden_states=return_hidden_states,
52
+ )
53
+
54
+ def prepare_model_input(
55
+ self,
56
+ seq_group_metadata_list: List[SequenceGroupMetadata],
57
+ virtual_engine: int = 0,
58
+ finished_requests_ids: Optional[List[str]] = None
59
+ ) -> ModelInputForGPUWithSamplingMetadata:
60
+ model_input: ModelInputForGPUWithSamplingMetadata = super(
61
+ ).prepare_model_input(seq_group_metadata_list, virtual_engine,
62
+ finished_requests_ids)
63
+ # If token log probabilities is disabled then skip generating sampler
64
+ # CPU output. We directly serialize the GPU sampled_token_id tensors
65
+ # as needed. If log probabilities is enabled then synchronize all the
66
+ # sampling related tensors which includes the logprobs tensors.
67
+ model_input.sampling_metadata.skip_sampler_cpu_output = (
68
+ self.disable_logprobs)
69
+ return model_input
vllm/spec_decode/util.py
+4
-4
@@ -1,5 +1,5 @@
1
from contextlib import contextmanager
2
-from typing import Dict, List, Tuple
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
@@ -53,8 +53,8 @@ def create_sequence_group_output(
53
token_id_logprob_rank: int,
54
token_id_logprob: float,
55
seq_id: SeqId,
56
- topk_token_ids: List[int],
57
- topk_logprobs: List[float],
56
+ topk_token_ids: List[Optional[int]],
57
+ topk_logprobs: List[Optional[float]],
58
) -> CompletionSequenceGroupOutput:
59
"""Create a SequenceGroupOutput given the sampling results.
60
@@ -68,7 +68,7 @@ def create_sequence_group_output(
68
"""
69
# vLLM logprobs always include the sampled token. In addition, the user may
70
# request topk-logprobs (where top-k varies per user up to max_logprobs).
71
- logprobs: Dict[int, Logprob] = {
71
+ logprobs: Dict[Optional[int], Logprob] = {
72
token_id: Logprob(
73
logprob=token_id_logprob,
74
rank=token_id_logprob_rank,