A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Dynamic Spec Decoding] Minor fix for disabling speculative decoding (#5000)
Lily Liu committed
May 25, 2024 at 10:00 UTC
d5a16977729928a2ceafb5ec8764081f40f7cdff
3 files changed
+63
-11
tests/spec_decode/e2e/test_ngram_correctness.py
+41
@@ -170,3 +170,44 @@ def test_ngram_different_k(baseline_llm_generator, test_llm_generator,
170
batch_size,
171
max_output_len=output_len,
172
force_output_len=True)
173
+
174
+
175
+@pytest.mark.parametrize(
176
+ "common_llm_kwargs",
177
+ [{
178
+ "model": "JackFram/llama-68m",
179
+
180
+ # Skip cuda graph recording for fast test.
181
+ "enforce_eager": True,
182
+
183
+ # Required for spec decode.
184
+ "use_v2_block_manager": True
185
+ }])
186
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
187
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
188
+@pytest.mark.parametrize("test_llm_kwargs",
189
+ [{
190
+ "speculative_model": "[ngram]",
191
+ "num_speculative_tokens": 5,
192
+ "ngram_prompt_lookup_max": 3,
193
+ "speculative_disable_by_batch_size": 4
194
+ }])
195
+@pytest.mark.parametrize("batch_size", [1, 5])
196
+@pytest.mark.parametrize(
197
+ "output_len",
198
+ [
199
+ # Use smaller output len for fast test.
200
+ 32,
201
+ ])
202
+@pytest.mark.parametrize("seed", [1])
203
+def test_ngram_disable_queue(baseline_llm_generator, test_llm_generator,
204
+ batch_size: int, output_len: int):
205
+ """Verify that ngram speculative decoding produces exact equality
206
+ to without spec decode with many different values of k and
207
+ different ngram_prompt_lookup_max.
208
+ """
209
+ run_greedy_equality_correctness_test(baseline_llm_generator,
210
+ test_llm_generator,
211
+ batch_size,
212
+ max_output_len=output_len,
213
+ force_output_len=True)
tests/spec_decode/test_dynamic_spec_decode.py
+10
-6
@@ -1,4 +1,4 @@
1
-from unittest.mock import MagicMock
1
+from unittest.mock import MagicMock, patch
2
3
import pytest
4
import torch
@@ -13,9 +13,9 @@ from vllm.spec_decode.top1_proposer import Top1Proposer
13
from .utils import create_batch, mock_worker
14
15
16
-@pytest.mark.parametrize('queue_size', [2, 4])
17
-@pytest.mark.parametrize('batch_size', [1, 2, 3, 6])
18
-@pytest.mark.parametrize('k', [1, 2, 5, 7, 10])
16
+@pytest.mark.parametrize('queue_size', [4])
17
+@pytest.mark.parametrize('batch_size', [1])
18
+@pytest.mark.parametrize('k', [1])
19
@torch.inference_mode()
20
def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
21
"""Verify that speculative tokens are disabled when the batch size
@@ -42,8 +42,12 @@ def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
42
num_lookahead_slots=k,
43
running_queue_size=queue_size)
44
45
- with pytest.raises(ValueError, match=exception_secret):
46
- worker.execute_model(execute_model_req=execute_model_req)
45
+ if queue_size > disable_by_batch_size:
46
+ with patch.object(worker,
47
+ '_run_no_spec',
48
+ side_effect=ValueError(exception_secret)), \
49
+ pytest.raises(ValueError, match=exception_secret):
50
+ worker.execute_model(execute_model_req=execute_model_req)
51
52
# When the batch size is larger than the threshold,
53
# we expect no speculative tokens (0).
vllm/spec_decode/spec_decode_worker.py
+12
-5
@@ -273,10 +273,17 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
273
self._maybe_disable_speculative_tokens(
274
disable_all_speculation, execute_model_req.seq_group_metadata_list)
275
276
- # If no spec tokens, call the proposer and scorer workers normally.
277
- # Used for prefill.
276
+ # Speculative decoding is disabled in the following cases:
277
+ # 1. Prefill phase: Speculative decoding is not
278
+ # used during the prefill phase.
279
+ # 2. Auto-disable enabled: The running queue size exceeds
280
+ # the specified threshold.
281
+ # 3. No request: There are no requests in the batch.
282
+ # In any of these cases, the proposer and scorer workers
283
+ # are called normally.
284
if num_lookahead_slots == 0 or len(
279
- execute_model_req.seq_group_metadata_list) == 0:
285
+ execute_model_req.seq_group_metadata_list
286
+ ) == 0 or disable_all_speculation:
287
return self._run_no_spec(execute_model_req,
288
skip_proposer=disable_all_speculation)
289
@@ -316,8 +323,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
323
@nvtx_range("spec_decode_worker._run_no_spec")
324
def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
325
skip_proposer: bool) -> List[SamplerOutput]:
319
- """Run a prefill step, without any speculation. The input is sent to
320
- the proposer and scorer model so that the KV cache is consistent
326
+ """Run a single generation step without any speculation. The input is
327
+ sent to the proposer and scorer model so that the KV cache is consistent
328
between the two. When skip_proposer is True, the proposer model is
329
not called, meaning that the kv-cache in proposer for requests is not
330
updated, so they cannot enable spec decode in the rest decoding.