@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d5a1697772
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Dynamic Spec Decoding] Minor fix for disabling speculative decoding (#5000)

Lily Liu committed May 25, 2024 at 10:00 UTC d5a16977729928a2ceafb5ec8764081f40f7cdff
3 files changed +63 -11
tests/spec_decode/e2e/test_ngram_correctness.py
+41
@@ -170,3 +170,44 @@ def test_ngram_different_k(baseline_llm_generator, test_llm_generator,
170 batch_size,
171 max_output_len=output_len,
172 force_output_len=True)
173 +
174 +
175 +@pytest.mark.parametrize(
176 + "common_llm_kwargs",
177 + [{
178 + "model": "JackFram/llama-68m",
179 +
180 + # Skip cuda graph recording for fast test.
181 + "enforce_eager": True,
182 +
183 + # Required for spec decode.
184 + "use_v2_block_manager": True
185 + }])
186 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
187 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
188 +@pytest.mark.parametrize("test_llm_kwargs",
189 + [{
190 + "speculative_model": "[ngram]",
191 + "num_speculative_tokens": 5,
192 + "ngram_prompt_lookup_max": 3,
193 + "speculative_disable_by_batch_size": 4
194 + }])
195 +@pytest.mark.parametrize("batch_size", [1, 5])
196 +@pytest.mark.parametrize(
197 + "output_len",
198 + [
199 + # Use smaller output len for fast test.
200 + 32,
201 + ])
202 +@pytest.mark.parametrize("seed", [1])
203 +def test_ngram_disable_queue(baseline_llm_generator, test_llm_generator,
204 + batch_size: int, output_len: int):
205 + """Verify that ngram speculative decoding produces exact equality
206 + to without spec decode with many different values of k and
207 + different ngram_prompt_lookup_max.
208 + """
209 + run_greedy_equality_correctness_test(baseline_llm_generator,
210 + test_llm_generator,
211 + batch_size,
212 + max_output_len=output_len,
213 + force_output_len=True)
tests/spec_decode/test_dynamic_spec_decode.py
+10 -6
@@ -1,4 +1,4 @@
1 -from unittest.mock import MagicMock
1 +from unittest.mock import MagicMock, patch
2
3 import pytest
4 import torch
@@ -13,9 +13,9 @@ from vllm.spec_decode.top1_proposer import Top1Proposer
13 from .utils import create_batch, mock_worker
14
15
16 -@pytest.mark.parametrize('queue_size', [2, 4])
17 -@pytest.mark.parametrize('batch_size', [1, 2, 3, 6])
18 -@pytest.mark.parametrize('k', [1, 2, 5, 7, 10])
16 +@pytest.mark.parametrize('queue_size', [4])
17 +@pytest.mark.parametrize('batch_size', [1])
18 +@pytest.mark.parametrize('k', [1])
19 @torch.inference_mode()
20 def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
21 """Verify that speculative tokens are disabled when the batch size
@@ -42,8 +42,12 @@ def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
42 num_lookahead_slots=k,
43 running_queue_size=queue_size)
44
45 - with pytest.raises(ValueError, match=exception_secret):
46 - worker.execute_model(execute_model_req=execute_model_req)
45 + if queue_size > disable_by_batch_size:
46 + with patch.object(worker,
47 + '_run_no_spec',
48 + side_effect=ValueError(exception_secret)), \
49 + pytest.raises(ValueError, match=exception_secret):
50 + worker.execute_model(execute_model_req=execute_model_req)
51
52 # When the batch size is larger than the threshold,
53 # we expect no speculative tokens (0).
vllm/spec_decode/spec_decode_worker.py
+12 -5
@@ -273,10 +273,17 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
273 self._maybe_disable_speculative_tokens(
274 disable_all_speculation, execute_model_req.seq_group_metadata_list)
275
276 - # If no spec tokens, call the proposer and scorer workers normally.
277 - # Used for prefill.
276 + # Speculative decoding is disabled in the following cases:
277 + # 1. Prefill phase: Speculative decoding is not
278 + # used during the prefill phase.
279 + # 2. Auto-disable enabled: The running queue size exceeds
280 + # the specified threshold.
281 + # 3. No request: There are no requests in the batch.
282 + # In any of these cases, the proposer and scorer workers
283 + # are called normally.
284 if num_lookahead_slots == 0 or len(
279 - execute_model_req.seq_group_metadata_list) == 0:
285 + execute_model_req.seq_group_metadata_list
286 + ) == 0 or disable_all_speculation:
287 return self._run_no_spec(execute_model_req,
288 skip_proposer=disable_all_speculation)
289
@@ -316,8 +323,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
323 @nvtx_range("spec_decode_worker._run_no_spec")
324 def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
325 skip_proposer: bool) -> List[SamplerOutput]:
319 - """Run a prefill step, without any speculation. The input is sent to
320 - the proposer and scorer model so that the KV cache is consistent
326 + """Run a single generation step without any speculation. The input is
327 + sent to the proposer and scorer model so that the KV cache is consistent
328 between the two. When skip_proposer is True, the proposer model is
329 not called, meaning that the kv-cache in proposer for requests is not
330 updated, so they cannot enable spec decode in the rest decoding.