A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[BUGFIX] Raise an error for no draft token case when draft_tp>1 (#6369)
Woo-Yeon Lee committed
Jul 19, 2024 at 22:01 UTC
a921e863921721ecae8250e0543aba1920c7c53a
4 files changed
+85
-5
tests/spec_decode/e2e/test_integration_dist_tp4.py
+62
@@ -58,3 +58,65 @@ def test_draft_model_tp_lt_target_model_tp4(test_llm_generator,
58
batch_size,
59
max_output_len=32,
60
force_output_len=True)
61
+
62
+
63
+@pytest.mark.skipif(torch.cuda.device_count() < 4,
64
+ reason="Need at least 4 GPUs to run the test.")
65
+@pytest.mark.parametrize(
66
+ "common_llm_kwargs",
67
+ [{
68
+ "model": "JackFram/llama-160m",
69
+
70
+ # Skip cuda graph recording for fast test.
71
+ "enforce_eager": True,
72
+
73
+ # Required for spec decode.
74
+ "use_v2_block_manager": True,
75
+ "tensor_parallel_size": 4,
76
+
77
+ # Use AsyncLLM engine, so that the engine runs in its own process.
78
+ # Otherwise, since vLLM does not follow true SPMD, the test runner
79
+ # process will have both the engine and the rank0 worker. NCCL is not
80
+ # cleaned up properly, and its server host thread leaks, causing the
81
+ # second run of the test to fail with internal NCCL error.
82
+ "use_async": True,
83
+ }])
84
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
85
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
86
+@pytest.mark.parametrize(
87
+ "test_llm_kwargs",
88
+ [
89
+ {
90
+ "speculative_model": "JackFram/llama-68m",
91
+ "num_speculative_tokens": 5,
92
+
93
+ # Artificially limit the draft model max model len; this forces vLLM
94
+ # to skip speculation once the sequences grow beyond 32-k tokens.
95
+ "speculative_max_model_len": 32,
96
+ },
97
+ ])
98
+@pytest.mark.parametrize("batch_size", [8])
99
+@pytest.mark.parametrize(
100
+ "output_len",
101
+ [
102
+ # This must be a good bit larger than speculative_max_model_len so that
103
+ # we can test the case where all seqs are skipped, but still small to
104
+ # ensure fast test.
105
+ 64,
106
+ ])
107
+@pytest.mark.parametrize("seed", [1])
108
+def test_skip_speculation(baseline_llm_generator, test_llm_generator,
109
+ batch_size: int, output_len: int):
110
+ """Verify job failure with RuntimeError when all sequences skip speculation.
111
+ We do this by setting the max model len of the draft model to an
112
+ artificially low value, such that when the sequences grow beyond it, they
113
+ are skipped in speculative decoding.
114
+
115
+ TODO: fix it to pass without raising Error. (#5814)
116
+ """
117
+ with pytest.raises(RuntimeError):
118
+ run_greedy_equality_correctness_test(baseline_llm_generator,
119
+ test_llm_generator,
120
+ batch_size,
121
+ max_output_len=output_len,
122
+ force_output_len=True)
vllm/spec_decode/interfaces.py
+3
@@ -22,6 +22,9 @@ class SpeculativeProposals:
22
# The valid length of each proposal; can be zero.
23
proposal_lens: torch.Tensor
24
25
+ # A flag to mark that there's no available proposals
26
+ no_proposals: bool = False
27
+
28
def __repr__(self):
29
return (f"SpeculativeProposals("
30
f"proposal_token_ids={self.proposal_token_ids}, "
vllm/spec_decode/spec_decode_worker.py
+19
-4
@@ -109,6 +109,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
109
typical_acceptance_sampler_posterior_alpha: float,
110
) -> "SpecDecodeWorker":
111
112
+ allow_zero_draft_token_step = True
113
ngram_prompt_lookup_max = (
114
draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
115
ngram_prompt_lookup_min = (
@@ -133,6 +134,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
134
if draft_tp == 1:
135
draft_worker_kwargs[
136
"model_runner_cls"] = TP1DraftModelRunner
137
+ else:
138
+ allow_zero_draft_token_step = False
139
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
140
141
proposer_worker = SmallerTpProposerWorker.maybe_wrap_worker(
@@ -155,10 +158,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
158
logger.info("Configuring SpecDecodeWorker with sampler=%s",
159
type(spec_decode_sampler))
160
158
- return SpecDecodeWorker(proposer_worker,
159
- scorer_worker,
160
- disable_by_batch_size=disable_by_batch_size,
161
- spec_decode_sampler=spec_decode_sampler)
161
+ return SpecDecodeWorker(
162
+ proposer_worker,
163
+ scorer_worker,
164
+ disable_by_batch_size=disable_by_batch_size,
165
+ spec_decode_sampler=spec_decode_sampler,
166
+ allow_zero_draft_token_step=allow_zero_draft_token_step)
167
168
def __init__(
169
self,
@@ -167,6 +172,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
172
spec_decode_sampler: SpecDecodeBaseSampler,
173
metrics_collector: Optional[AsyncMetricsCollector] = None,
174
disable_by_batch_size: Optional[int] = None,
175
+ allow_zero_draft_token_step: Optional[bool] = True,
176
):
177
"""
178
Create a SpecDecodeWorker.
@@ -187,11 +193,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
193
disable speculative decoding for new incoming requests.
194
metrics_collector: Helper class for collecting metrics; can be set
195
for testing purposes.
196
+ allow_zero_draft_token_step: whether to allow a step where the draft
197
+ model generates no draft token; should disallow when the tp of
198
+ draft model is larger than 1 (TODO: #5814)
199
"""
200
self.proposer_worker = proposer_worker
201
self.scorer_worker = scorer_worker
202
self.disable_by_batch_size = disable_by_batch_size or float("inf")
203
self.spec_decode_sampler = spec_decode_sampler
204
+ self._allow_zero_draft_token_step = allow_zero_draft_token_step
205
self._metrics = AsyncMetricsCollector(
206
self.spec_decode_sampler
207
) if metrics_collector is None else metrics_collector
@@ -461,6 +471,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
471
proposals = self.proposer_worker.get_spec_proposals(
472
execute_model_req, self._seq_with_bonus_token_in_last_step)
473
474
+ if not self._allow_zero_draft_token_step and proposals.no_proposals:
475
+ #TODO: Fix it #5814
476
+ raise RuntimeError("Cannot handle cases where distributed draft "
477
+ "workers generate no tokens")
478
+
479
proposal_scores = self.scorer.score_proposals(
480
execute_model_req,
481
proposals,
vllm/spec_decode/top1_proposer.py
+1
-1
@@ -108,7 +108,7 @@ class Top1Proposer(SpeculativeProposer):
108
proposal_token_ids=proposal_tokens,
109
proposal_probs=proposal_probs,
110
proposal_lens=proposal_lens,
111
- )
111
+ no_proposals=maybe_sampler_output is None)
112
113
return proposals
114