@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a921e86392
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[BUGFIX] Raise an error for no draft token case when draft_tp>1 (#6369)

Woo-Yeon Lee committed Jul 19, 2024 at 22:01 UTC a921e863921721ecae8250e0543aba1920c7c53a
4 files changed +85 -5
tests/spec_decode/e2e/test_integration_dist_tp4.py
+62
@@ -58,3 +58,65 @@ def test_draft_model_tp_lt_target_model_tp4(test_llm_generator,
58 batch_size,
59 max_output_len=32,
60 force_output_len=True)
61 +
62 +
63 +@pytest.mark.skipif(torch.cuda.device_count() < 4,
64 + reason="Need at least 4 GPUs to run the test.")
65 +@pytest.mark.parametrize(
66 + "common_llm_kwargs",
67 + [{
68 + "model": "JackFram/llama-160m",
69 +
70 + # Skip cuda graph recording for fast test.
71 + "enforce_eager": True,
72 +
73 + # Required for spec decode.
74 + "use_v2_block_manager": True,
75 + "tensor_parallel_size": 4,
76 +
77 + # Use AsyncLLM engine, so that the engine runs in its own process.
78 + # Otherwise, since vLLM does not follow true SPMD, the test runner
79 + # process will have both the engine and the rank0 worker. NCCL is not
80 + # cleaned up properly, and its server host thread leaks, causing the
81 + # second run of the test to fail with internal NCCL error.
82 + "use_async": True,
83 + }])
84 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
85 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
86 +@pytest.mark.parametrize(
87 + "test_llm_kwargs",
88 + [
89 + {
90 + "speculative_model": "JackFram/llama-68m",
91 + "num_speculative_tokens": 5,
92 +
93 + # Artificially limit the draft model max model len; this forces vLLM
94 + # to skip speculation once the sequences grow beyond 32-k tokens.
95 + "speculative_max_model_len": 32,
96 + },
97 + ])
98 +@pytest.mark.parametrize("batch_size", [8])
99 +@pytest.mark.parametrize(
100 + "output_len",
101 + [
102 + # This must be a good bit larger than speculative_max_model_len so that
103 + # we can test the case where all seqs are skipped, but still small to
104 + # ensure fast test.
105 + 64,
106 + ])
107 +@pytest.mark.parametrize("seed", [1])
108 +def test_skip_speculation(baseline_llm_generator, test_llm_generator,
109 + batch_size: int, output_len: int):
110 + """Verify job failure with RuntimeError when all sequences skip speculation.
111 + We do this by setting the max model len of the draft model to an
112 + artificially low value, such that when the sequences grow beyond it, they
113 + are skipped in speculative decoding.
114 +
115 + TODO: fix it to pass without raising Error. (#5814)
116 + """
117 + with pytest.raises(RuntimeError):
118 + run_greedy_equality_correctness_test(baseline_llm_generator,
119 + test_llm_generator,
120 + batch_size,
121 + max_output_len=output_len,
122 + force_output_len=True)
vllm/spec_decode/interfaces.py
+3
@@ -22,6 +22,9 @@ class SpeculativeProposals:
22 # The valid length of each proposal; can be zero.
23 proposal_lens: torch.Tensor
24
25 + # A flag to mark that there's no available proposals
26 + no_proposals: bool = False
27 +
28 def __repr__(self):
29 return (f"SpeculativeProposals("
30 f"proposal_token_ids={self.proposal_token_ids}, "
vllm/spec_decode/spec_decode_worker.py
+19 -4
@@ -109,6 +109,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
109 typical_acceptance_sampler_posterior_alpha: float,
110 ) -> "SpecDecodeWorker":
111
112 + allow_zero_draft_token_step = True
113 ngram_prompt_lookup_max = (
114 draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
115 ngram_prompt_lookup_min = (
@@ -133,6 +134,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
134 if draft_tp == 1:
135 draft_worker_kwargs[
136 "model_runner_cls"] = TP1DraftModelRunner
137 + else:
138 + allow_zero_draft_token_step = False
139 proposer_worker = MultiStepWorker(**draft_worker_kwargs)
140
141 proposer_worker = SmallerTpProposerWorker.maybe_wrap_worker(
@@ -155,10 +158,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
158 logger.info("Configuring SpecDecodeWorker with sampler=%s",
159 type(spec_decode_sampler))
160
158 - return SpecDecodeWorker(proposer_worker,
159 - scorer_worker,
160 - disable_by_batch_size=disable_by_batch_size,
161 - spec_decode_sampler=spec_decode_sampler)
161 + return SpecDecodeWorker(
162 + proposer_worker,
163 + scorer_worker,
164 + disable_by_batch_size=disable_by_batch_size,
165 + spec_decode_sampler=spec_decode_sampler,
166 + allow_zero_draft_token_step=allow_zero_draft_token_step)
167
168 def __init__(
169 self,
@@ -167,6 +172,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
172 spec_decode_sampler: SpecDecodeBaseSampler,
173 metrics_collector: Optional[AsyncMetricsCollector] = None,
174 disable_by_batch_size: Optional[int] = None,
175 + allow_zero_draft_token_step: Optional[bool] = True,
176 ):
177 """
178 Create a SpecDecodeWorker.
@@ -187,11 +193,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
193 disable speculative decoding for new incoming requests.
194 metrics_collector: Helper class for collecting metrics; can be set
195 for testing purposes.
196 + allow_zero_draft_token_step: whether to allow a step where the draft
197 + model generates no draft token; should disallow when the tp of
198 + draft model is larger than 1 (TODO: #5814)
199 """
200 self.proposer_worker = proposer_worker
201 self.scorer_worker = scorer_worker
202 self.disable_by_batch_size = disable_by_batch_size or float("inf")
203 self.spec_decode_sampler = spec_decode_sampler
204 + self._allow_zero_draft_token_step = allow_zero_draft_token_step
205 self._metrics = AsyncMetricsCollector(
206 self.spec_decode_sampler
207 ) if metrics_collector is None else metrics_collector
@@ -461,6 +471,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
471 proposals = self.proposer_worker.get_spec_proposals(
472 execute_model_req, self._seq_with_bonus_token_in_last_step)
473
474 + if not self._allow_zero_draft_token_step and proposals.no_proposals:
475 + #TODO: Fix it #5814
476 + raise RuntimeError("Cannot handle cases where distributed draft "
477 + "workers generate no tokens")
478 +
479 proposal_scores = self.scorer.score_proposals(
480 execute_model_req,
481 proposals,
vllm/spec_decode/top1_proposer.py
+1 -1
@@ -108,7 +108,7 @@ class Top1Proposer(SpeculativeProposer):
108 proposal_token_ids=proposal_tokens,
109 proposal_probs=proposal_probs,
110 proposal_lens=proposal_lens,
111 - )
111 + no_proposals=maybe_sampler_output is None)
112
113 return proposals
114