@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82a1b1a82b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Speculative decoding] Add periodic log with time spent in proposal/scoring/verification (#6963)

Cade Daniel committed Aug 5, 2024 at 01:46 UTC 82a1b1a82b1fbb454c82a9ef95730b929c9b270c
5 files changed +125 -35
tests/spec_decode/test_spec_decode_worker.py
+48 -20
@@ -34,8 +34,11 @@ def test_correctly_calls_draft_model(k: int, batch_size: int,
34 target_worker = mock_worker()
35 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
36 worker = SpecDecodeWorker(
37 - draft_worker, target_worker,
38 - mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
37 + draft_worker,
38 + target_worker,
39 + mock_spec_decode_sampler(acceptance_sampler_method),
40 + disable_logprobs=False,
41 + metrics_collector=metrics_collector)
42 exception_secret = 'artificial stop'
43 draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
44
@@ -74,8 +77,11 @@ def test_correctly_calls_target_model(k: int, batch_size: int,
77 set_random_seed(1)
78
79 worker = SpecDecodeWorker(
77 - draft_worker, target_worker,
78 - mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
80 + draft_worker,
81 + target_worker,
82 + mock_spec_decode_sampler(acceptance_sampler_method),
83 + disable_logprobs=False,
84 + metrics_collector=metrics_collector)
85 worker.init_device()
86
87 vocab_size = 32_000
@@ -159,8 +165,11 @@ def test_correctly_calls_spec_decode_sampler(k: int, batch_size: int,
165
166 set_random_seed(1)
167
162 - worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
163 - metrics_collector)
168 + worker = SpecDecodeWorker(draft_worker,
169 + target_worker,
170 + spec_decode_sampler,
171 + disable_logprobs=False,
172 + metrics_collector=metrics_collector)
173 worker.init_device()
174
175 proposal_token_ids = torch.randint(low=0,
@@ -249,8 +258,11 @@ def test_correctly_formats_output(k: int, batch_size: int,
258
259 set_random_seed(1)
260 spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
252 - worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
253 - metrics_collector)
261 + worker = SpecDecodeWorker(draft_worker,
262 + target_worker,
263 + spec_decode_sampler,
264 + disable_logprobs=False,
265 + metrics_collector=metrics_collector)
266 worker.init_device()
267
268 proposal_token_ids = torch.randint(low=0,
@@ -479,9 +491,13 @@ def test_k_equals_zero(k: int, batch_size: int,
491 set_random_seed(1)
492
493 worker = SpecDecodeWorker(
482 - draft_worker, target_worker,
483 - mock_spec_decode_sampler(acceptance_sampler_method), False,
484 - metrics_collector)
494 + proposer_worker=draft_worker,
495 + scorer_worker=target_worker,
496 + spec_decode_sampler=mock_spec_decode_sampler(
497 + acceptance_sampler_method),
498 + disable_logprobs=False,
499 + metrics_collector=metrics_collector,
500 + )
501
502 seq_group_metadata_list, _, _ = create_batch(batch_size,
503 k,
@@ -526,9 +542,13 @@ def test_empty_input_batch(k: int, batch_size: int,
542 set_random_seed(1)
543
544 worker = SpecDecodeWorker(
529 - draft_worker, target_worker,
530 - mock_spec_decode_sampler(acceptance_sampler_method), False,
531 - metrics_collector)
545 + proposer_worker=draft_worker,
546 + scorer_worker=target_worker,
547 + spec_decode_sampler=mock_spec_decode_sampler(
548 + acceptance_sampler_method),
549 + disable_logprobs=False,
550 + metrics_collector=metrics_collector,
551 + )
552
553 seq_group_metadata_list, _, _ = create_batch(batch_size,
554 k,
@@ -560,8 +580,13 @@ def test_init_device(acceptance_sampler_method: str):
580 spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
581 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
582
563 - worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
564 - False, metrics_collector)
583 + worker = SpecDecodeWorker(
584 + proposer_worker=draft_worker,
585 + scorer_worker=target_worker,
586 + spec_decode_sampler=spec_decode_sampler,
587 + disable_logprobs=False,
588 + metrics_collector=metrics_collector,
589 + )
590 worker.init_device()
591
592 draft_worker.init_device.assert_called_once()
@@ -583,9 +608,11 @@ def test_initialize_cache(acceptance_sampler_method):
608 target_worker = mock_worker()
609 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
610
586 - worker = SpecDecodeWorker(
587 - draft_worker, target_worker,
588 - mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
611 + worker = SpecDecodeWorker(proposer_worker=draft_worker,
612 + scorer_worker=target_worker,
613 + spec_decode_sampler=mock_spec_decode_sampler(
614 + acceptance_sampler_method),
615 + metrics_collector=metrics_collector)
616
617 kwargs = {"num_gpu_blocks": 1024, "num_cpu_blocks": 1023}
618 worker.initialize_cache(**kwargs)
@@ -725,7 +752,8 @@ def test_populate_seq_ids_with_bonus_tokens():
752 seq_group_metadata_list=seq_group_metadata_list,
753 accepted_token_ids=accepted_token_ids,
754 target_logprobs=target_token_logprobs,
728 - k=k)
755 + k=k,
756 + stage_times=(0, 0, 0))
757 # Verify that _seq_with_bonus_token_in_last_step contains the following:
758 # 1. Sequence IDs that were already present in
759 # _seq_with_bonus_token_in_last_step but were not part of the current
vllm/config.py
+7 -1
@@ -907,6 +907,7 @@ class SpeculativeConfig:
907 speculative_max_model_len: Optional[int],
908 enable_chunked_prefill: bool,
909 use_v2_block_manager: bool,
910 + disable_log_stats: bool,
911 speculative_disable_by_batch_size: Optional[int],
912 ngram_prompt_lookup_max: Optional[int],
913 ngram_prompt_lookup_min: Optional[int],
@@ -1095,7 +1096,8 @@ class SpeculativeConfig:
1096 typical_acceptance_sampler_posterior_threshold,
1097 typical_acceptance_sampler_posterior_alpha=\
1098 typical_acceptance_sampler_posterior_alpha,
1098 - disable_logprobs=disable_logprobs
1099 + disable_logprobs=disable_logprobs,
1100 + disable_log_stats=disable_log_stats,
1101 )
1102
1103 @staticmethod
@@ -1189,6 +1191,7 @@ class SpeculativeConfig:
1191 typical_acceptance_sampler_posterior_threshold: float,
1192 typical_acceptance_sampler_posterior_alpha: float,
1193 disable_logprobs: bool,
1194 + disable_log_stats: bool,
1195 ):
1196 """Create a SpeculativeConfig object.
1197
@@ -1221,6 +1224,8 @@ class SpeculativeConfig:
1224 sampling, target sampling, and after accepted tokens are
1225 determined. If set to False, log probabilities will be
1226 returned.
1227 + disable_log_stats: Whether to disable periodic printing of stage
1228 + times in speculative decoding.
1229 """
1230 self.draft_model_config = draft_model_config
1231 self.draft_parallel_config = draft_parallel_config
@@ -1235,6 +1240,7 @@ class SpeculativeConfig:
1240 self.typical_acceptance_sampler_posterior_alpha = \
1241 typical_acceptance_sampler_posterior_alpha
1242 self.disable_logprobs = disable_logprobs
1243 + self.disable_log_stats = disable_log_stats
1244
1245 self._verify_args()
1246
vllm/engine/arg_utils.py
+1
@@ -792,6 +792,7 @@ class EngineArgs:
792 speculative_max_model_len=self.speculative_max_model_len,
793 enable_chunked_prefill=self.enable_chunked_prefill,
794 use_v2_block_manager=self.use_v2_block_manager,
795 + disable_log_stats=self.disable_log_stats,
796 ngram_prompt_lookup_max=self.ngram_prompt_lookup_max,
797 ngram_prompt_lookup_min=self.ngram_prompt_lookup_min,
798 draft_token_acceptance_method=\
vllm/spec_decode/spec_decode_worker.py
+54 -14
@@ -27,7 +27,7 @@ from vllm.spec_decode.ngram_worker import NGramWorker
27 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
28 from vllm.spec_decode.smaller_tp_proposer_worker import SmallerTpProposerWorker
29 from vllm.spec_decode.target_model_runner import TargetModelRunner
30 -from vllm.spec_decode.util import (create_sequence_group_output,
30 +from vllm.spec_decode.util import (Timer, create_sequence_group_output,
31 get_all_num_logprobs,
32 get_sampled_token_logprobs, nvtx_range,
33 split_batch_by_proposal_len)
@@ -75,7 +75,9 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
75 typical_acceptance_sampler_posterior_threshold,
76 typical_acceptance_sampler_posterior_alpha=speculative_config.
77 typical_acceptance_sampler_posterior_alpha,
78 - disable_logprobs=speculative_config.disable_logprobs)
78 + disable_logprobs=speculative_config.disable_logprobs,
79 + disable_log_stats=speculative_config.disable_log_stats,
80 + )
81
82 return spec_decode_worker
83
@@ -116,6 +118,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
118 typical_acceptance_sampler_posterior_threshold: float,
119 typical_acceptance_sampler_posterior_alpha: float,
120 disable_logprobs: bool,
121 + disable_log_stats: bool,
122 ) -> "SpecDecodeWorker":
123
124 allow_zero_draft_token_step = True
@@ -171,6 +174,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
174 proposer_worker,
175 scorer_worker,
176 disable_logprobs=disable_logprobs,
177 + disable_log_stats=disable_log_stats,
178 disable_by_batch_size=disable_by_batch_size,
179 spec_decode_sampler=spec_decode_sampler,
180 allow_zero_draft_token_step=allow_zero_draft_token_step)
@@ -180,7 +184,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
184 proposer_worker: ProposerWorkerBase,
185 scorer_worker: WorkerBase,
186 spec_decode_sampler: SpecDecodeBaseSampler,
183 - disable_logprobs: bool,
187 + disable_logprobs: bool = False,
188 + disable_log_stats: bool = False,
189 metrics_collector: Optional[AsyncMetricsCollector] = None,
190 disable_by_batch_size: Optional[int] = None,
191 allow_zero_draft_token_step: Optional[bool] = True,
@@ -203,6 +208,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
208 disable_logprobs: If set to True, token log probabilities will
209 not be output in both the draft worker and the target worker.
210 If set to False, log probabilities will be output by both.
211 + disable_log_stats: If set to True, disable periodic printing of
212 + speculative stage times.
213 disable_by_batch_size: If the batch size is larger than this,
214 disable speculative decoding for new incoming requests.
215 metrics_collector: Helper class for collecting metrics; can be set
@@ -240,6 +247,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
247 # in the subsequent step.
248 self.previous_hidden_states: Optional[HiddenStates] = None
249 self._disable_logprobs = disable_logprobs
250 + self._disable_log_stats = disable_log_stats
251
252 def init_device(self) -> None:
253 """Initialize both scorer and proposer models.
@@ -525,28 +533,37 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
533 execute_model_req.previous_hidden_states = self.previous_hidden_states
534 self.previous_hidden_states = None
535
528 - # Generate proposals using draft worker.
529 - proposals = self.proposer_worker.get_spec_proposals(
530 - execute_model_req, self._seq_with_bonus_token_in_last_step)
536 + with Timer() as proposal_timer:
537 + # Generate proposals using draft worker.
538 + proposals = self.proposer_worker.get_spec_proposals(
539 + execute_model_req, self._seq_with_bonus_token_in_last_step)
540
541 if not self._allow_zero_draft_token_step and proposals.no_proposals:
542 #TODO: Fix it #5814
543 raise RuntimeError("Cannot handle cases where distributed draft "
544 "workers generate no tokens")
545
537 - proposal_scores = self.scorer.score_proposals(
538 - execute_model_req,
539 - proposals,
540 - )
541 - accepted_token_ids, target_logprobs = self._verify_tokens(
542 - execute_model_req.seq_group_metadata_list, proposal_scores,
543 - proposals, execute_model_req.num_lookahead_slots)
546 + with Timer() as scoring_timer:
547 + proposal_scores = self.scorer.score_proposals(
548 + execute_model_req,
549 + proposals,
550 + )
551 +
552 + with Timer() as verification_timer:
553 + accepted_token_ids, target_logprobs = self._verify_tokens(
554 + execute_model_req.seq_group_metadata_list, proposal_scores,
555 + proposals, execute_model_req.num_lookahead_slots)
556 +
557 + stage_times = (proposal_timer.elapsed_time_ms / num_lookahead_slots,
558 + scoring_timer.elapsed_time_ms,
559 + verification_timer.elapsed_time_ms)
560
561 return self._create_output_sampler_list(
562 execute_model_req.seq_group_metadata_list,
563 accepted_token_ids,
564 target_logprobs=target_logprobs,
549 - k=execute_model_req.num_lookahead_slots)
565 + k=execute_model_req.num_lookahead_slots,
566 + stage_times=stage_times)
567
568 @nvtx_range("spec_decode_worker._verify_tokens")
569 def _verify_tokens(
@@ -645,6 +662,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
662 accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1]
663 target_logprobs: torch.Tensor, # shape: [batch_size, k+1, vocab_size]
664 k: int,
665 + stage_times: Tuple[float, float, float],
666 ) -> List[SamplerOutput]:
667 """Given the accepted token ids, create a list of SamplerOutput.
668
@@ -722,8 +740,30 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
740 if maybe_rejsample_metrics is not None:
741 sampler_output_list[
742 0].spec_decode_worker_metrics = maybe_rejsample_metrics
743 +
744 + # Log time spent in each stage periodically.
745 + # This is periodic because the rejection sampler emits metrics
746 + # periodically.
747 + self._maybe_log_stage_times(*stage_times)
748 +
749 return sampler_output_list
750
751 + def _maybe_log_stage_times(self, average_time_per_proposal_tok_ms: float,
752 + scoring_time_ms: float,
753 + verification_time_ms: float) -> None:
754 + """Log the speculative stage times. If stat logging is disabled, do
755 + nothing.
756 + """
757 + if self._disable_log_stats:
758 + return
759 +
760 + logger.info(
761 + "SpecDecodeWorker stage times: "
762 + "average_time_per_proposal_tok_ms=%.02f "
763 + "scoring_time_ms=%.02f verification_time_ms=%.02f",
764 + average_time_per_proposal_tok_ms, scoring_time_ms,
765 + verification_time_ms)
766 +
767 def _create_dummy_logprob_lists(
768 self,
769 batch_size: int,
vllm/spec_decode/util.py
+15
@@ -1,3 +1,4 @@
1 +import time
2 from contextlib import contextmanager
3 from typing import Dict, List, Optional, Tuple
4
@@ -214,3 +215,17 @@ def nvtx_range(msg, *args, **kwargs):
215 yield
216 finally:
217 torch.cuda.nvtx.range_pop()
218 +
219 +
220 +class Timer:
221 + """Basic timer context manager for measuring CPU time.
222 + """
223 +
224 + def __enter__(self):
225 + self.start_time = time.time()
226 + return self
227 +
228 + def __exit__(self, exc_type, exc_value, traceback):
229 + self.end_time = time.time()
230 + self.elapsed_time_s = self.end_time - self.start_time
231 + self.elapsed_time_ms = self.elapsed_time_s * 1000