@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b38e42fbca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Speculative decoding] Add ngram prompt lookup decoding (#4237)

Co-authored-by: Lei Wen <wenlei03@qiyi.com>

leiwen83 committed May 2, 2024 at 02:13 UTC b38e42fbca978d62cc8330bdcf8da91c72cb2ebc
14 files changed +1004 -319
tests/spec_decode/e2e/conftest.py
+58
@@ -1,4 +1,5 @@
1 import asyncio
2 +from itertools import cycle
3 from typing import List, Optional, Tuple, Union
4
5 import pytest
@@ -185,3 +186,60 @@ def get_output_from_llm_generator(
186 del llm
187
188 return tokens, token_ids
189 +
190 +
191 +def run_greedy_equality_correctness_test(baseline_llm_generator,
192 + test_llm_generator,
193 + batch_size,
194 + max_output_len,
195 + force_output_len: bool,
196 + print_tokens: bool = False):
197 + """Helper method that compares the outputs of both the baseline LLM and
198 + the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
199 + the same when temperature is zero.
200 + """
201 + temperature = 0.0
202 +
203 + prompts = [
204 + "Hello, my name is",
205 + "The president of the United States is",
206 + "The capital of France is",
207 + "The future of AI is",
208 + "San Francisco is know for its",
209 + "Facebook was created in 2004 by",
210 + "Curious George is a",
211 + "Python 3.11 brings improvements to its",
212 + ]
213 +
214 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
215 +
216 + # If the test requires that we generated max_output_len tokens, then set the
217 + # sampling params to ignore eos token.
218 + ignore_eos = force_output_len
219 +
220 + sampling_params = SamplingParams(
221 + max_tokens=max_output_len,
222 + ignore_eos=ignore_eos,
223 + temperature=temperature,
224 + )
225 +
226 + spec_batch_tokens, spec_batch_token_ids = get_output_from_llm_generator(
227 + test_llm_generator, prompts, sampling_params)
228 +
229 + (baseline_batch_tokens,
230 + baseline_batch_token_ids) = get_output_from_llm_generator(
231 + baseline_llm_generator, prompts, sampling_params)
232 +
233 + assert len(baseline_batch_token_ids) == len(prompts)
234 + assert len(spec_batch_token_ids) == len(prompts)
235 +
236 + for i, (baseline_token_ids, baseline_tokens, spec_token_ids,
237 + spec_tokens) in enumerate(
238 + zip(baseline_batch_token_ids, baseline_batch_tokens,
239 + spec_batch_token_ids, spec_batch_tokens)):
240 + if print_tokens:
241 + print(f'{i=} {baseline_tokens=}')
242 + print(f'{i=} {spec_tokens=}')
243 + print(f'{i=} {baseline_token_ids=}')
244 + print(f'{i=} {spec_token_ids=}')
245 + assert baseline_token_ids == spec_token_ids
tests/spec_decode/e2e/test_multistep_correctness.py renamed
+2 -58
@@ -35,7 +35,8 @@ from transformers import AutoTokenizer
35
36 from vllm import SamplingParams
37
38 -from .conftest import get_output_from_llm_generator
38 +from .conftest import (get_output_from_llm_generator,
39 + run_greedy_equality_correctness_test)
40
41
42 @pytest.mark.parametrize(
@@ -545,60 +546,3 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
546 batch_size,
547 max_output_len=output_len,
548 force_output_len=True)
548 -
549 -
550 -def run_greedy_equality_correctness_test(baseline_llm_generator,
551 - test_llm_generator,
552 - batch_size,
553 - max_output_len,
554 - force_output_len: bool,
555 - print_tokens: bool = False):
556 - """Helper method that compares the outputs of both the baseline LLM and
557 - the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
558 - the same when temperature is zero.
559 - """
560 - temperature = 0.0
561 -
562 - prompts = [
563 - "Hello, my name is",
564 - "The president of the United States is",
565 - "The capital of France is",
566 - "The future of AI is",
567 - "San Francisco is know for its",
568 - "Facebook was created in 2004 by",
569 - "Curious George is a",
570 - "Python 3.11 brings improvements to its",
571 - ]
572 -
573 - prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
574 -
575 - # If the test requires that we generated max_output_len tokens, then set the
576 - # sampling params to ignore eos token.
577 - ignore_eos = force_output_len
578 -
579 - sampling_params = SamplingParams(
580 - max_tokens=max_output_len,
581 - ignore_eos=ignore_eos,
582 - temperature=temperature,
583 - )
584 -
585 - spec_batch_tokens, spec_batch_token_ids = get_output_from_llm_generator(
586 - test_llm_generator, prompts, sampling_params)
587 -
588 - (baseline_batch_tokens,
589 - baseline_batch_token_ids) = get_output_from_llm_generator(
590 - baseline_llm_generator, prompts, sampling_params)
591 -
592 - assert len(baseline_batch_token_ids) == len(prompts)
593 - assert len(spec_batch_token_ids) == len(prompts)
594 -
595 - for i, (baseline_token_ids, baseline_tokens, spec_token_ids,
596 - spec_tokens) in enumerate(
597 - zip(baseline_batch_token_ids, baseline_batch_tokens,
598 - spec_batch_token_ids, spec_batch_tokens)):
599 - if print_tokens:
600 - print(f'{i=} {baseline_tokens=}')
601 - print(f'{i=} {spec_tokens=}')
602 - print(f'{i=} {baseline_token_ids=}')
603 - print(f'{i=} {spec_token_ids=}')
604 - assert baseline_token_ids == spec_token_ids
tests/spec_decode/e2e/test_ngram_correctness.py new
+172
@@ -0,0 +1,172 @@
1 +"""This docstring details important information on the testing methodology.
2 +
3 +Most of the tests rely on "greedy equality", where we expect the output of
4 +speculative decoding on a sequence to exactly match the output of normal non-
5 +speculative decoding.
6 +
7 +Since speculative decoding with rejection sampling guarantees that the output
8 +distribution matches the target model's output distribution (up to hardware
9 +numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
10 +equality.
11 +
12 +For ngram lookup, its idea comes from https://github.com/apoorvumang/prompt-lookup-decoding,
13 +and is merged into transform code base: https://github.com/huggingface/transformers/pull/27775.
14 +Since there is no model is needed for generate the proposal, we could make
15 +the testcase much simpler than drafter multi-step one.
16 +
17 +However, we still need to verify below scenario could be passed:
18 + * Batch size 1 greedy equality
19 + * Batch size >1 greedy equality
20 + * Test greedy equality under preemption
21 + * Test greedy equality under various ngram sizes / speculative sizes
22 +
23 +With those tests, we can say at least, ngram spec would not break the correctess
24 +for the target model outputs.
25 +"""
26 +
27 +import pytest
28 +
29 +from .conftest import run_greedy_equality_correctness_test
30 +
31 +
32 +@pytest.mark.parametrize(
33 + "common_llm_kwargs",
34 + [{
35 + # Skip cuda graph recording for fast test.
36 + "enforce_eager": True,
37 +
38 + # Required for spec decode.
39 + "use_v2_block_manager": True,
40 +
41 + # Print spec metrics.
42 + "disable_log_stats": False,
43 + }])
44 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
45 + {
46 + "model": "JackFram/llama-68m",
47 + },
48 +])
49 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
50 +@pytest.mark.parametrize("test_llm_kwargs", [
51 + {
52 + "speculative_model": "[ngram]",
53 + "num_speculative_tokens": 5,
54 + "ngram_prompt_lookup_max": 3,
55 + },
56 +])
57 +@pytest.mark.parametrize("output_len", [
58 + 256,
59 +])
60 +@pytest.mark.parametrize("batch_size", [1, 64])
61 +@pytest.mark.parametrize("seed", [1])
62 +def test_ngram_e2e_greedy_correctness(baseline_llm_generator,
63 + test_llm_generator, batch_size: int,
64 + output_len: int):
65 + """Verify greedy equality on a tiny model with different batch size."""
66 + run_greedy_equality_correctness_test(baseline_llm_generator,
67 + test_llm_generator,
68 + batch_size,
69 + max_output_len=output_len,
70 + force_output_len=True)
71 +
72 +
73 +@pytest.mark.parametrize(
74 + "common_llm_kwargs",
75 + [{
76 + "block_size": 8,
77 + # 2 for small prompt, 256//8 for generated.
78 + "num_gpu_blocks_override": 2 + 256 // 8,
79 + "max_model_len": (2 + 256 // 8) * 8,
80 +
81 + # Skip cuda graph recording for fast test.
82 + "enforce_eager": True,
83 +
84 + # Required for spec decode.
85 + "use_v2_block_manager": True
86 + }])
87 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
88 + {
89 + "model": "JackFram/llama-160m",
90 + },
91 +])
92 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
93 +@pytest.mark.parametrize("test_llm_kwargs", [
94 + {
95 + "speculative_model": "[ngram]",
96 + "num_speculative_tokens": 5,
97 + "ngram_prompt_lookup_max": 3,
98 + },
99 +])
100 +@pytest.mark.parametrize(
101 + "output_len",
102 + [
103 + # Use small output len for fast test.
104 + 256,
105 + ])
106 +@pytest.mark.parametrize("batch_size", [4])
107 +@pytest.mark.parametrize("seed", [1])
108 +def test_ngram_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
109 + test_llm_generator,
110 + batch_size: int,
111 + output_len: int):
112 + """Verify greedy equality, even when some sequences are preempted mid-
113 + generation.
114 + """
115 + run_greedy_equality_correctness_test(baseline_llm_generator,
116 + test_llm_generator,
117 + batch_size,
118 + max_output_len=output_len,
119 + force_output_len=True)
120 +
121 +
122 +@pytest.mark.parametrize(
123 + "common_llm_kwargs",
124 + [{
125 + "model": "JackFram/llama-68m",
126 +
127 + # Skip cuda graph recording for fast test.
128 + "enforce_eager": True,
129 +
130 + # Required for spec decode.
131 + "use_v2_block_manager": True
132 + }])
133 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
134 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
135 +@pytest.mark.parametrize(
136 + "test_llm_kwargs",
137 + [
138 + {
139 + "speculative_model": "[ngram]",
140 + "num_speculative_tokens": k,
141 + "ngram_prompt_lookup_max": 3,
142 + }
143 + # Try a range of common k, as well as large speculation.
144 + for k in [1, 3, 5]
145 + ] + [
146 + {
147 + "speculative_model": "[ngram]",
148 + "num_speculative_tokens": k,
149 + "ngram_prompt_lookup_max": 1,
150 + }
151 + # Try a range of common k, as well as large speculation.
152 + for k in [1, 3, 5]
153 + ])
154 +@pytest.mark.parametrize("batch_size", [2])
155 +@pytest.mark.parametrize(
156 + "output_len",
157 + [
158 + # Use smaller output len for fast test.
159 + 32,
160 + ])
161 +@pytest.mark.parametrize("seed", [1])
162 +def test_ngram_different_k(baseline_llm_generator, test_llm_generator,
163 + batch_size: int, output_len: int):
164 + """Verify that ngram speculative decoding produces exact equality
165 + to without spec decode with many different values of k and
166 + different ngram_prompt_lookup_max.
167 + """
168 + run_greedy_equality_correctness_test(baseline_llm_generator,
169 + test_llm_generator,
170 + batch_size,
171 + max_output_len=output_len,
172 + force_output_len=True)
tests/spec_decode/test_multi_step_worker.py
+25 -25
@@ -6,8 +6,8 @@ import torch
6
7 from vllm.model_executor.utils import set_random_seed
8 from vllm.sequence import SamplerOutput
9 -from vllm.spec_decode.multi_step_worker import (DraftModelTop1Proposer,
10 - MultiStepWorker)
9 +from vllm.spec_decode.multi_step_worker import MultiStepWorker
10 +from vllm.spec_decode.top1_proposer import Top1Proposer
11 from vllm.worker.worker import Worker
12
13 from .utils import (assert_logprobs_dict_allclose, create_batch,
@@ -117,8 +117,8 @@ def test_same_output_for_single_step():
117
118 zero_kv_cache(multi_step_worker.cache_engine)
119 set_random_seed(seed)
120 - actual_output = multi_step_worker.execute_model_multi_step(
121 - **multi_step_execute_model_data.to_dict(), num_steps=num_steps)
120 + actual_output, _ = multi_step_worker.sampler_output(
121 + **multi_step_execute_model_data.to_dict(), sample_len=num_steps)
122 assert len(actual_output) == num_steps
123 actual_output = actual_output[0]
124
@@ -200,8 +200,8 @@ def test_same_output_for_multi_step():
200 # Run multi-step.
201 zero_kv_cache(multi_step_worker.cache_engine)
202 set_random_seed(seed)
203 - multi_step_output = multi_step_worker.execute_model_multi_step(
204 - **execute_model_data.to_dict(), num_steps=num_steps)
203 + multi_step_output, _ = multi_step_worker.sampler_output(
204 + **execute_model_data.to_dict(), sample_len=num_steps)
205
206 # Run single-step repeatedly.
207 zero_kv_cache(worker.cache_engine)
@@ -266,7 +266,7 @@ def test_same_output_for_multi_step():
266
267 @torch.inference_mode()
268 def test_draft_proposals_full_speculation_len():
269 - """Verify DraftModelTop1Proposer correctly handles case where all sequences
269 + """Verify Top1Proposer correctly handles case where all sequences
270 can speculate.
271 """
272 k = 10
@@ -275,13 +275,13 @@ def test_draft_proposals_full_speculation_len():
275 device = 'cuda:0'
276
277 draft_worker = MagicMock()
278 - proposer = DraftModelTop1Proposer(
279 - draft_worker=draft_worker,
278 + proposer = Top1Proposer(
279 + worker=draft_worker,
280 device=device,
281 - max_model_len=2048,
281 vocab_size=vocab_size,
282 + max_proposal_len=2048,
283 )
284 - draft_worker.execute_model_multi_step.return_value = [
284 + draft_worker.sampler_output.return_value = [
285 SamplerOutput(
286 outputs=[],
287 sampled_token_probs=torch.rand(batch_size,
@@ -294,13 +294,13 @@ def test_draft_proposals_full_speculation_len():
294 device=device,
295 dtype=torch.long),
296 ) for _ in range(k)
297 - ]
297 + ], True
298
299 execute_model_data, _, _ = create_batch(batch_size, k)
300
301 proposals = proposer.get_proposals(
302 **execute_model_data.to_dict(),
303 - max_proposal_len=k,
303 + proposal_len=k,
304 )
305
306 assert torch.is_tensor(proposals.proposal_token_ids)
@@ -315,7 +315,7 @@ def test_draft_proposals_full_speculation_len():
315
316 @torch.inference_mode()
317 def test_draft_proposals_no_speculations():
318 - """Verify DraftModelTop1Proposer correctly handles case where no sequences
318 + """Verify Top1Proposer correctly handles case where no sequences
319 can speculate.
320 """
321 k = 10
@@ -325,11 +325,11 @@ def test_draft_proposals_no_speculations():
325 prompt_len = 10
326
327 draft_worker = MagicMock()
328 - proposer = DraftModelTop1Proposer(
329 - draft_worker=draft_worker,
328 + proposer = Top1Proposer(
329 + worker=draft_worker,
330 device=device,
331 - max_model_len=prompt_len + k - 1,
331 vocab_size=vocab_size,
332 + max_proposal_len=prompt_len + k - 1,
333 )
334
335 execute_model_data, _, _ = create_batch(batch_size,
@@ -338,7 +338,7 @@ def test_draft_proposals_no_speculations():
338
339 proposals = proposer.get_proposals(
340 **execute_model_data.to_dict(),
341 - max_proposal_len=k,
341 + proposal_len=k,
342 )
343
344 assert torch.is_tensor(proposals.proposal_token_ids)
@@ -353,7 +353,7 @@ def test_draft_proposals_no_speculations():
353
354 @torch.inference_mode()
355 def test_draft_proposals_mixed_k():
356 - """Verify DraftModelTop1Proposer correctly handles case some sequences can
356 + """Verify Top1Proposer correctly handles case some sequences can
357 speculate and some can't.
358 """
359 k = 10
@@ -374,14 +374,14 @@ def test_draft_proposals_mixed_k():
374 for _ in range(expected_num_no_proposal_seqs)] + [small_prompt_len]
375
376 draft_worker = MagicMock()
377 - proposer = DraftModelTop1Proposer(
378 - draft_worker=draft_worker,
377 + proposer = Top1Proposer(
378 + worker=draft_worker,
379 device=device,
380 - max_model_len=long_prompt_len + prev_output_token_len + k - 1,
380 vocab_size=vocab_size,
381 + max_proposal_len=long_prompt_len + prev_output_token_len + k - 1,
382 )
383
384 - draft_worker.execute_model_multi_step.return_value = [
384 + draft_worker.sampler_output.return_value = [
385 SamplerOutput(
386 outputs=[],
387 sampled_token_probs=torch.rand(expected_num_proposal_seqs,
@@ -395,7 +395,7 @@ def test_draft_proposals_mixed_k():
395 device=device,
396 dtype=torch.long),
397 ) for _ in range(k)
398 - ]
398 + ], True
399
400 execute_model_data, _, _ = create_batch(
401 batch_size,
@@ -406,7 +406,7 @@ def test_draft_proposals_mixed_k():
406
407 proposals = proposer.get_proposals(
408 **execute_model_data.to_dict(),
409 - max_proposal_len=k,
409 + proposal_len=k,
410 )
411
412 assert torch.is_tensor(proposals.proposal_token_ids)
tests/spec_decode/test_ngram_worker.py new
+206
@@ -0,0 +1,206 @@
1 +import torch
2 +
3 +from vllm.spec_decode.ngram_worker import NGramWorker
4 +from vllm.spec_decode.top1_proposer import Top1Proposer
5 +
6 +from .utils import (create_execute_model_data,
7 + create_seq_group_metadata_from_prompts, create_worker)
8 +
9 +
10 +def test_ngram_algo_correctness_for_single_no_match():
11 + """Verify our ngram algo find the right candidate in the prompt
12 +
13 + For the scenario cannot find any candidate in one single batch
14 + """
15 + block_size = 32
16 + num_gpu_blocks = 2048 // block_size
17 + seed = 100
18 + model_name = 'JackFram/llama-68m'
19 + vocab_size = 32_000
20 + device = 'cuda:0'
21 +
22 + ngram_worker = create_worker(
23 + NGramWorker,
24 + model_name,
25 + block_size,
26 + num_gpu_blocks,
27 + seed,
28 + )
29 +
30 + proposer = Top1Proposer(
31 + worker=ngram_worker,
32 + device=device,
33 + vocab_size=vocab_size,
34 + max_proposal_len=20,
35 + )
36 +
37 + # set ngram window (0, 3], which is window=1/2/3
38 + ngram_worker.set_ngram_window_size(0, 3)
39 +
40 + prompts = [
41 + # shall find no candidate
42 + [1, 2, 3, 4, 5, 6, 7],
43 + ]
44 +
45 + proposal_len = 5
46 + final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
47 + ngram_sampler_output_data = create_execute_model_data(
48 + seq_group_metadata_list=create_seq_group_metadata_from_prompts(
49 + prompts, num_gpu_blocks, block_size,
50 + final_seq_lens=final_seq_lens))
51 +
52 + proposals = proposer.get_proposals(
53 + **ngram_sampler_output_data.to_dict(),
54 + proposal_len=proposal_len,
55 + )
56 +
57 + assert torch.is_tensor(proposals.proposal_token_ids)
58 + assert torch.is_tensor(proposals.proposal_probs)
59 +
60 + assert proposals.proposal_token_ids.shape == torch.Size([1, proposal_len])
61 + assert proposals.proposal_probs.shape[:-1] == torch.Size([1, proposal_len])
62 + assert proposals.proposal_lens.shape == torch.Size([1])
63 + assert proposals.proposal_lens.tolist() == [0]
64 +
65 +
66 +def test_ngram_algo_correctness_for_batches_not_match_all():
67 + """Verify our ngram algo find the right candidate in the prompt
68 +
69 + For the scenario find some candidate not full in batchs
70 + """
71 + block_size = 32
72 + num_gpu_blocks = 2048 // block_size
73 + seed = 100
74 + model_name = 'JackFram/llama-68m'
75 + vocab_size = 32_000
76 + device = 'cuda:0'
77 +
78 + ngram_worker = create_worker(
79 + NGramWorker,
80 + model_name,
81 + block_size,
82 + num_gpu_blocks,
83 + seed,
84 + )
85 +
86 + proposer = Top1Proposer(
87 + worker=ngram_worker,
88 + device=device,
89 + vocab_size=vocab_size,
90 + max_proposal_len=20,
91 + )
92 +
93 + # set ngram window (0, 3], which is window=1/2/3
94 + ngram_worker.set_ngram_window_size(0, 3)
95 +
96 + prompts = [
97 + # shall find no candidate
98 + [1, 2, 3, 4, 5, 6, 7],
99 + # shall find candidate 12,13,14,15,16
100 + [11, 12, 13, 14, 15, 16, 11],
101 + # shall find candidate 23,24,25,26,21
102 + [21, 21, 22, 23, 24, 25, 26, 21, 22],
103 + # shall find candidate 34,35,36,37,38
104 + [31, 32, 31, 32, 33, 34, 35, 36, 37, 38, 31, 32, 33],
105 + # shall find no candidate as exceed max_proposal_len
106 + [
107 + 31, 32, 31, 32, 31, 32, 31, 32, 31, 32, 31, 32, 33, 34, 35, 36, 37,
108 + 38, 31, 32, 33
109 + ],
110 + ]
111 +
112 + proposal_len = 5
113 + final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
114 + ngram_sampler_output_data = create_execute_model_data(
115 + seq_group_metadata_list=create_seq_group_metadata_from_prompts(
116 + prompts, num_gpu_blocks, block_size,
117 + final_seq_lens=final_seq_lens))
118 +
119 + proposals = proposer.get_proposals(
120 + **ngram_sampler_output_data.to_dict(),
121 + proposal_len=proposal_len,
122 + )
123 +
124 + assert torch.is_tensor(proposals.proposal_token_ids)
125 + assert torch.is_tensor(proposals.proposal_probs)
126 +
127 + assert proposals.proposal_token_ids.shape == torch.Size([5, proposal_len])
128 + assert proposals.proposal_probs.shape[:-1] == torch.Size([5, proposal_len])
129 + assert proposals.proposal_lens.shape == torch.Size([5])
130 +
131 + assert proposals.proposal_lens.tolist(
132 + ) == [proposal_len for _ in range(4)] + [0]
133 +
134 + for i in range(proposal_len):
135 + assert proposals.proposal_token_ids[0][i] == 0
136 + assert proposals.proposal_token_ids[1][i] == prompts[1][i + 1]
137 + assert proposals.proposal_token_ids[2][i] == prompts[2][i + 3]
138 + assert proposals.proposal_token_ids[3][i] == prompts[3][i + 5]
139 + assert proposals.proposal_token_ids[4][i] == -1
140 +
141 +
142 +def test_ngram_algo_correctness_for_batches_match_all():
143 + """Verify our ngram algo find the right candidate in the prompt
144 +
145 + For the scenario find candidate in all batchs
146 + """
147 +
148 + block_size = 32
149 + num_gpu_blocks = 2048 // block_size
150 + seed = 100
151 + model_name = 'JackFram/llama-68m'
152 + vocab_size = 32_000
153 + device = 'cuda:0'
154 +
155 + ngram_worker = create_worker(
156 + NGramWorker,
157 + model_name,
158 + block_size,
159 + num_gpu_blocks,
160 + seed,
161 + )
162 +
163 + proposer = Top1Proposer(
164 + worker=ngram_worker,
165 + device=device,
166 + vocab_size=vocab_size,
167 + max_proposal_len=20,
168 + )
169 +
170 + # set ngram window (0, 3], which is window=1/2/3
171 + ngram_worker.set_ngram_window_size(0, 3)
172 +
173 + prompts = [
174 + # shall find candidate 12,13,14,15,16
175 + [11, 12, 13, 14, 15, 16, 11],
176 + # shall find candidate 23,24,25,26,21
177 + [21, 21, 22, 23, 24, 25, 26, 21, 22],
178 + # shall find candidate 34,35,36,37,38
179 + [31, 32, 31, 32, 33, 34, 35, 36, 37, 38, 31, 32, 33],
180 + ]
181 +
182 + proposal_len = 5
183 + final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
184 + ngram_sampler_output_data = create_execute_model_data(
185 + seq_group_metadata_list=create_seq_group_metadata_from_prompts(
186 + prompts, num_gpu_blocks, block_size,
187 + final_seq_lens=final_seq_lens))
188 +
189 + proposals = proposer.get_proposals(
190 + **ngram_sampler_output_data.to_dict(),
191 + proposal_len=proposal_len,
192 + )
193 +
194 + assert torch.is_tensor(proposals.proposal_token_ids)
195 + assert torch.is_tensor(proposals.proposal_probs)
196 +
197 + assert proposals.proposal_token_ids.shape == torch.Size([3, proposal_len])
198 + assert proposals.proposal_probs.shape[:-1] == torch.Size([3, proposal_len])
199 + assert proposals.proposal_lens.shape == torch.Size([3])
200 +
201 + assert proposals.proposal_lens.tolist() == [proposal_len for _ in range(3)]
202 +
203 + for i in range(proposal_len):
204 + assert proposals.proposal_token_ids[0][i] == prompts[0][i + 1]
205 + assert proposals.proposal_token_ids[1][i] == prompts[1][i + 3]
206 + assert proposals.proposal_token_ids[2][i] == prompts[2][i + 5]
vllm/config.py
+59 -28
@@ -682,6 +682,8 @@ class SpeculativeConfig:
682 speculative_max_model_len: Optional[int],
683 enable_chunked_prefill: bool,
684 use_v2_block_manager: bool,
685 + ngram_prompt_lookup_max: Optional[int],
686 + ngram_prompt_lookup_min: Optional[int],
687 ) -> Optional["SpeculativeConfig"]:
688 """Create a SpeculativeConfig if possible, else return None.
689
@@ -708,6 +710,10 @@ class SpeculativeConfig:
710 use_v2_block_manager (bool): Whether vLLM is configured to use the
711 v2 block manager or not. Used for raising an error since the v2
712 block manager is required with spec decode.
713 + ngram_prompt_lookup_max (Optional[int]): Max size of ngram token
714 + window, if provided.
715 + ngram_prompt_lookup_min (Optional[int]): Min size of ngram token
716 + window, if provided.
717
718 Returns:
719 Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
@@ -742,39 +748,57 @@ class SpeculativeConfig:
748 draft_code_revision = None
749 draft_quantization = None
750
745 - draft_model_config = ModelConfig(
746 - model=speculative_model,
747 - tokenizer=target_model_config.tokenizer,
748 - tokenizer_mode=target_model_config.tokenizer_mode,
749 - trust_remote_code=target_model_config.trust_remote_code,
750 - dtype=target_model_config.dtype,
751 - seed=target_model_config.seed,
752 - revision=draft_revision,
753 - code_revision=draft_code_revision,
754 - tokenizer_revision=target_model_config.tokenizer_revision,
755 - max_model_len=None,
756 - quantization=draft_quantization,
757 - enforce_eager=target_model_config.enforce_eager,
758 - max_context_len_to_capture=target_model_config.
759 - max_context_len_to_capture,
760 - max_logprobs=target_model_config.max_logprobs,
761 - )
762 -
763 - draft_model_config.max_model_len = (
764 - SpeculativeConfig._maybe_override_draft_max_model_len(
765 - speculative_max_model_len,
766 - draft_model_config.max_model_len,
767 - target_model_config.max_model_len,
768 - ))
751 + if speculative_model == "[ngram]":
752 + assert (ngram_prompt_lookup_max is not None
753 + and ngram_prompt_lookup_max > 0)
754 + if ngram_prompt_lookup_min is None:
755 + ngram_prompt_lookup_min = 0
756 + else:
757 + assert ngram_prompt_lookup_max > ngram_prompt_lookup_min
758
770 - draft_parallel_config = (
771 - SpeculativeConfig.create_draft_parallel_config(
772 - target_parallel_config))
759 + # TODO: current we still need extract vocab_size from target model
760 + # config, in future, we may try refactor it out, and set
761 + # draft related config as None here.
762 + draft_model_config = target_model_config
763 + draft_parallel_config = target_parallel_config
764 + else:
765 + ngram_prompt_lookup_max = 0
766 + ngram_prompt_lookup_min = 0
767 + draft_model_config = ModelConfig(
768 + model=speculative_model,
769 + tokenizer=target_model_config.tokenizer,
770 + tokenizer_mode=target_model_config.tokenizer_mode,
771 + trust_remote_code=target_model_config.trust_remote_code,
772 + dtype=target_model_config.dtype,
773 + seed=target_model_config.seed,
774 + revision=draft_revision,
775 + code_revision=draft_code_revision,
776 + tokenizer_revision=target_model_config.tokenizer_revision,
777 + max_model_len=None,
778 + quantization=draft_quantization,
779 + enforce_eager=target_model_config.enforce_eager,
780 + max_context_len_to_capture=target_model_config.
781 + max_context_len_to_capture,
782 + max_logprobs=target_model_config.max_logprobs,
783 + )
784 +
785 + draft_model_config.max_model_len = (
786 + SpeculativeConfig._maybe_override_draft_max_model_len(
787 + speculative_max_model_len,
788 + draft_model_config.max_model_len,
789 + target_model_config.max_model_len,
790 + ))
791 +
792 + draft_parallel_config = (
793 + SpeculativeConfig.create_draft_parallel_config(
794 + target_parallel_config))
795
796 return SpeculativeConfig(
797 draft_model_config,
798 draft_parallel_config,
799 num_speculative_tokens,
800 + ngram_prompt_lookup_max,
801 + ngram_prompt_lookup_min,
802 )
803
804 @staticmethod
@@ -842,6 +866,8 @@ class SpeculativeConfig:
866 draft_model_config: ModelConfig,
867 draft_parallel_config: ParallelConfig,
868 num_speculative_tokens: int,
869 + ngram_prompt_lookup_max: int,
870 + ngram_prompt_lookup_min: int,
871 ):
872 """Create a SpeculativeConfig object.
873
@@ -854,6 +880,8 @@ class SpeculativeConfig:
880 self.draft_model_config = draft_model_config
881 self.draft_parallel_config = draft_parallel_config
882 self.num_speculative_tokens = num_speculative_tokens
883 + self.ngram_prompt_lookup_max = ngram_prompt_lookup_max
884 + self.ngram_prompt_lookup_min = ngram_prompt_lookup_min
885
886 self._verify_args()
887
@@ -877,7 +905,10 @@ class SpeculativeConfig:
905 return self.num_speculative_tokens
906
907 def __repr__(self) -> str:
880 - draft_model = self.draft_model_config.model
908 + if self.ngram_prompt_lookup_max > 0:
909 + draft_model = "[ngram]"
910 + else:
911 + draft_model = self.draft_model_config.model
912 num_spec_tokens = self.num_speculative_tokens
913 return f"SpeculativeConfig({draft_model=}, {num_spec_tokens=})"
914
vllm/engine/arg_utils.py
+18
@@ -75,6 +75,8 @@ class EngineArgs:
75 speculative_model: Optional[str] = None
76 num_speculative_tokens: Optional[int] = None
77 speculative_max_model_len: Optional[int] = None
78 + ngram_prompt_lookup_max: Optional[int] = None
79 + ngram_prompt_lookup_min: Optional[int] = None
80
81 def __post_init__(self):
82 if self.tokenizer is None:
@@ -449,6 +451,20 @@ class EngineArgs:
451 'draft model. Sequences over this length will skip '
452 'speculation.')
453
454 + parser.add_argument(
455 + '--ngram-prompt-lookup-max',
456 + type=int,
457 + default=EngineArgs.ngram_prompt_lookup_max,
458 + help='Max size of window for ngram prompt lookup in speculative '
459 + 'decoding.')
460 +
461 + parser.add_argument(
462 + '--ngram-prompt-lookup-min',
463 + type=int,
464 + default=EngineArgs.ngram_prompt_lookup_min,
465 + help='Min size of window for ngram prompt lookup in speculative '
466 + 'decoding.')
467 +
468 parser.add_argument('--model-loader-extra-config',
469 type=str,
470 default=EngineArgs.model_loader_extra_config,
@@ -502,6 +518,8 @@ class EngineArgs:
518 speculative_max_model_len=self.speculative_max_model_len,
519 enable_chunked_prefill=self.enable_chunked_prefill,
520 use_v2_block_manager=self.use_v2_block_manager,
521 + ngram_prompt_lookup_max=self.ngram_prompt_lookup_max,
522 + ngram_prompt_lookup_min=self.ngram_prompt_lookup_min,
523 )
524
525 scheduler_config = SchedulerConfig(
vllm/executor/gpu_executor.py
+4 -4
@@ -73,7 +73,6 @@ class GPUExecutor(ExecutorBase):
73 """
74 assert self.speculative_config is not None
75
76 - from vllm.spec_decode.multi_step_worker import MultiStepWorker
76 from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
77
78 target_worker = self._create_worker()
@@ -86,10 +85,11 @@ class GPUExecutor(ExecutorBase):
85 # TODO allow draft-model specific load config.
86 #load_config=self.load_config,
87 )
89 - draft_worker = MultiStepWorker(**draft_worker_kwargs)
88
91 - spec_decode_worker = SpecDecodeWorker.from_workers(
92 - proposer_worker=draft_worker, scorer_worker=target_worker)
89 + spec_decode_worker = SpecDecodeWorker.create_worker(
90 + scorer_worker=target_worker,
91 + draft_worker_kwargs=draft_worker_kwargs,
92 + )
93
94 assert self.parallel_config.world_size == 1, (
95 "GPUExecutor only supports single GPU.")
vllm/spec_decode/batch_expansion.py
+2 -2
@@ -333,13 +333,13 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
333 sampler_output.sampled_token_probs = spec_probs
334 sampler_output.sampled_token_ids = spec_sampled_tokens
335 target_token_ids, target_probs = sampler_output_to_torch(
336 - [sampler_output])
336 + [sampler_output], True)
337
338 # Convert non-speculative output tokens to tensors.
339 sampler_output.sampled_token_probs = non_spec_probs
340 sampler_output.sampled_token_ids = non_spec_sampled_tokens
341 non_spec_target_token_ids, non_spec_target_probs = (
342 - sampler_output_to_torch([sampler_output]))
342 + sampler_output_to_torch([sampler_output], True))
343
344 return (target_token_ids, target_probs, non_spec_target_token_ids,
345 non_spec_target_probs)
vllm/spec_decode/multi_step_worker.py
+24 -185
@@ -1,12 +1,11 @@
1 import copy
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import Dict, List, Tuple
3
4 import torch
5
6 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 -from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 - SpeculativeProposer)
9 -from vllm.spec_decode.util import sampler_output_to_torch
7 +from vllm.spec_decode.interfaces import SpeculativeProposals
8 +from vllm.spec_decode.top1_proposer import Top1Proposer
9 from vllm.worker.worker import Worker
10
11
@@ -26,29 +25,37 @@ class MultiStepWorker(Worker):
25 super().__init__(*args, **kwargs)
26
27 # Lazy initialization list.
29 - self._proposer: DraftModelTop1Proposer
28 + self._proposer: Top1Proposer
29
30 def init_device(self):
31 super().init_device()
32
34 - self._proposer = DraftModelTop1Proposer(
33 + self._proposer = Top1Proposer(
34 self,
35 self.device,
37 - self.max_model_len,
36 self.vocab_size,
37 + max_proposal_len=self.max_model_len,
38 )
39
40 + def set_include_gpu_probs_tensor(self):
41 + # Need include_gpu_probs_tensor for multi_step_worker
42 + self.model_runner.model.sampler.include_gpu_probs_tensor = True
43 +
44 @torch.inference_mode()
42 - def execute_model_multi_step(
45 + def sampler_output(
46 self,
47 seq_group_metadata_list: List[SequenceGroupMetadata],
48 blocks_to_swap_in: Dict[int, int],
49 blocks_to_swap_out: Dict[int, int],
50 blocks_to_copy: Dict[int, List[int]],
48 - num_steps: int,
49 - ) -> List[SamplerOutput]:
50 - """Run the model forward pass num_steps times. Returns the list of
51 - sampler output, one per model forward pass.
51 + sample_len: int,
52 + ) -> Tuple[List[SamplerOutput], bool]:
53 + """Run the model forward pass sample_len times. Returns the list of
54 + sampler output, one per model forward pass, along with indicator of
55 + whether torch tensor in sampler output need to be transposed in latter
56 + sampler_output_to_torch logic.
57 +
58 + For multi step worker, this indicator shall be True.
59 """
60 self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
61 blocks_to_swap_out, blocks_to_copy)
@@ -58,12 +65,12 @@ class MultiStepWorker(Worker):
65 copied_seq_group_metadata_list = self._shallow_copy_inputs(
66 seq_group_metadata_list)
67
61 - # Assert enough KV space for num_steps tokens per sequence.
62 - self._assert_enough_kv_space(seq_group_metadata_list, num_steps)
68 + # Assert enough KV space for sample_len tokens per sequence.
69 + self._assert_enough_kv_space(seq_group_metadata_list, sample_len)
70
64 - # Run model num_steps times.
71 + # Run model sample_len times.
72 model_outputs = []
66 - for _ in range(num_steps):
73 + for _ in range(sample_len):
74 model_output = super().execute_model(
75 seq_group_metadata_list=copied_seq_group_metadata_list,
76 blocks_to_swap_in=blocks_to_swap_in,
@@ -78,7 +85,7 @@ class MultiStepWorker(Worker):
85 copied_seq_group_metadata_list)
86 model_outputs.append(model_output)
87
81 - return model_outputs
88 + return model_outputs, True
89
90 def get_spec_proposals(
91 self,
@@ -206,171 +213,3 @@ class MultiStepWorker(Worker):
213 for seq_group_metadata in seq_group_metadata_list):
214 raise NotImplementedError(
215 "MultiStepWorker does not support beam search.")
209 -
210 -
211 -class DraftModelTop1Proposer(SpeculativeProposer):
212 - """Helper class which separates out sequences which would exceed the max
213 - model length when speculated upon.
214 -
215 - This allows combinations of models such as JackFram/llama-68m draft with
216 - meta-llama/Llama2-13b-chat-hf, as llama-68m has max_position_embeddings of
217 - 2048 while Llama2-13b has max_position_embeddings of 4096.
218 -
219 - We treat the sequences which exceed the proposal draft model length as
220 - "non-spec sequences". Essentially they skip the draft model and go through
221 - normal decoding in the target model.
222 -
223 - Currently, only proposal_lens of 0 and k are supported, where k is a global
224 - batch proposal length. In the future vLLM should support per-sequence
225 - proposal lengths.
226 - """
227 -
228 - def __init__(
229 - self,
230 - draft_worker: MultiStepWorker,
231 - device: str,
232 - max_model_len: int,
233 - vocab_size: int,
234 - ):
235 - self._draft_worker = draft_worker
236 - self._device = device
237 - self._max_model_len = max_model_len
238 - self._vocab_size = vocab_size
239 -
240 - def get_proposals(
241 - self,
242 - seq_group_metadata_list: List[SequenceGroupMetadata],
243 - blocks_to_swap_in: Dict[int, int],
244 - blocks_to_swap_out: Dict[int, int],
245 - blocks_to_copy: Dict[int, List[int]],
246 - max_proposal_len: int,
247 - ) -> SpeculativeProposals:
248 - """Get speculative proposals given the input batch.
249 -
250 - Sequences which would exceed the max model length are skipped during
251 - speculation.
252 - """
253 -
254 - # Split speculative- and non-speculative- sequences.
255 - (proposal_lens, nonzero_proposal_len_seqs,
256 - nonzero_proposal_len_indices) = self._split_by_max_model_len(
257 - seq_group_metadata_list, max_proposal_len)
258 -
259 - if nonzero_proposal_len_seqs:
260 - # Speculate tokens using the draft worker for the speculative
261 - # sequences.
262 - maybe_sampler_output = self._draft_worker.execute_model_multi_step(
263 - seq_group_metadata_list=nonzero_proposal_len_seqs,
264 - blocks_to_swap_in=blocks_to_swap_in,
265 - blocks_to_swap_out=blocks_to_swap_out,
266 - blocks_to_copy=blocks_to_copy,
267 - num_steps=max_proposal_len,
268 - )
269 - else:
270 - # If no sequences can be speculated, set sampler output to None.
271 - maybe_sampler_output = None
272 -
273 - # Combine speculative- and non-speculative sequences into the same
274 - # representation.
275 - proposal_tokens, proposal_probs, proposal_lens = self._merge_outputs(
276 - batch_size=len(seq_group_metadata_list),
277 - max_proposal_len=max_proposal_len,
278 - maybe_sampler_output=maybe_sampler_output,
279 - proposal_lens=proposal_lens,
280 - nonzero_proposal_len_indices=nonzero_proposal_len_indices,
281 - )
282 -
283 - proposals = SpeculativeProposals(
284 - proposal_token_ids=proposal_tokens,
285 - proposal_probs=proposal_probs,
286 - proposal_lens=proposal_lens,
287 - )
288 -
289 - return proposals
290 -
291 - def _split_by_max_model_len(
292 - self,
293 - seq_group_metadata_list: List[SequenceGroupMetadata],
294 - max_proposal_len: int,
295 - ) -> Tuple[List[int], List[SequenceGroupMetadata], List[int]]:
296 - """Determine which sequences would exceed the max model length.
297 - """
298 -
299 - proposal_lens: List[int] = []
300 - nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
301 - nonzero_proposal_len_indices: List[int] = []
302 - for i, seq_group_metadata in enumerate(seq_group_metadata_list):
303 - seq_data = next(iter(seq_group_metadata.seq_data.values()))
304 - seq_len = seq_data.get_len()
305 -
306 - # Currently only proposal lens of 0 or the global batch proposal len
307 - # are supported.
308 - if seq_len + max_proposal_len < self._max_model_len:
309 - proposal_lens.append(max_proposal_len)
310 - nonzero_proposal_len_seqs.append(seq_group_metadata)
311 - nonzero_proposal_len_indices.append(i)
312 - else:
313 - proposal_lens.append(0)
314 -
315 - return (proposal_lens, nonzero_proposal_len_seqs,
316 - nonzero_proposal_len_indices)
317 -
318 - def _merge_outputs(
319 - self,
320 - batch_size: int,
321 - max_proposal_len: int,
322 - maybe_sampler_output: Optional[SamplerOutput],
323 - proposal_lens: List[int],
324 - nonzero_proposal_len_indices: List[int],
325 - ) -> Tuple[torch.Tensor, torch.tensor, torch.Tensor]:
326 - """After speculations are produced, merge the speculation results with
327 - the skipped sequences.
328 - """
329 - if maybe_sampler_output is None:
330 - # If no speculative tokens, the sampler output will be None.
331 - # In this case we return empty proposals.
332 - proposal_tokens = torch.full(size=(
333 - batch_size,
334 - max_proposal_len,
335 - ),
336 - fill_value=-1,
337 - dtype=torch.long,
338 - device=self._device)
339 - proposal_probs = torch.zeros(batch_size,
340 - max_proposal_len,
341 - self._vocab_size,
342 - dtype=torch.float32,
343 - device=self._device)
344 - proposal_lens_tensor = torch.zeros(len(proposal_lens),
345 - dtype=torch.long,
346 - device=self._device)
347 - return proposal_tokens, proposal_probs, proposal_lens_tensor
348 -
349 - sampler_output = maybe_sampler_output
350 - proposal_tokens, proposal_probs = sampler_output_to_torch(
351 - sampler_output)
352 -
353 - # Now, reformat the output GPU tensors such that each sequence has
354 - # a proposal. the proposal can be empty, e.g. [-1, -1, -1]
355 -
356 - entire_proposal_tokens = torch.full(size=(batch_size,
357 - *proposal_tokens.shape[1:]),
358 - fill_value=-1,
359 - dtype=torch.long,
360 - device=self._device)
361 - entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens
362 - entire_proposal_probs = torch.zeros(batch_size,
363 - *proposal_probs.shape[1:],
364 - dtype=torch.float32,
365 - device=self._device)
366 - entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
367 -
368 - proposal_tokens, proposal_probs = (entire_proposal_tokens,
369 - entire_proposal_probs)
370 -
371 - proposal_lens_tensor = torch.zeros(batch_size,
372 - dtype=torch.long,
373 - device=self._device)
374 - proposal_lens_tensor[nonzero_proposal_len_indices] = max_proposal_len
375 -
376 - return proposal_tokens, proposal_probs, proposal_lens_tensor
vllm/spec_decode/ngram_worker.py new
+190
@@ -0,0 +1,190 @@
1 +from typing import Dict, List, Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6 +from vllm.spec_decode.interfaces import SpeculativeProposals
7 +from vllm.spec_decode.top1_proposer import Top1Proposer
8 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase
9 +
10 +
11 +class NGramWorker(LoraNotSupportedWorkerBase):
12 + """NGramWorker provides a light drafter without need for model.
13 +
14 + Current NGramWorker only implement prompt lookup decoding,
15 + and in future we may also do RAG type drafter and other scenerios
16 + which don't rely on LLM model to give proposals.
17 + """
18 +
19 + def __init__(self, *args, **kwargs):
20 + # Get local_rank/vocab_size from kwargs attribute
21 + self.local_rank = kwargs["local_rank"]
22 + self.vocab_size = kwargs["model_config"].get_vocab_size()
23 +
24 + # Lazy initialization list.
25 + self._proposer: Top1Proposer
26 +
27 + def set_ngram_window_size(self, ngram_prompt_lookup_min: int,
28 + ngram_prompt_lookup_max: int):
29 + # Search valid candidate window between
30 + # ngram_prompt_lookup_min/ngram_prompt_lookup_max
31 + self.ngram_prompt_lookup_max = ngram_prompt_lookup_max
32 + self.ngram_prompt_lookup_min = ngram_prompt_lookup_min
33 +
34 + def init_device(self):
35 + self.device = torch.device(f"cuda:{self.local_rank}")
36 + self.load_model = lambda *args, **kwargs: None
37 +
38 + # Current only support Top1Proposer
39 + self._proposer = Top1Proposer(
40 + self,
41 + device=self.device,
42 + vocab_size=self.vocab_size,
43 + )
44 +
45 + def set_include_gpu_probs_tensor(self):
46 + # NGram don't need gpu sampler
47 + pass
48 +
49 + def execute_model(
50 + self,
51 + seq_group_metadata_list: List[SequenceGroupMetadata],
52 + blocks_to_swap_in: Optional[Dict[int, int]],
53 + blocks_to_swap_out: Optional[Dict[int, int]],
54 + blocks_to_copy: Optional[Dict[int, List[int]]],
55 + ) -> None:
56 + """NGram doesn't depend on model execution, just pass this function"""
57 + pass
58 +
59 + def determine_num_available_blocks(self) -> None:
60 + """NGram doesn't depend on model execution, no need to check blocks"""
61 + pass
62 +
63 + def initialize_cache(self, num_gpu_blocks: int,
64 + num_cpu_blocks: int) -> None:
65 + """As there is no cache need to handle, just pass this function"""
66 + pass
67 +
68 + def get_cache_block_size_bytes(self):
69 + """Return the size of a cache block in bytes."""
70 + return 0
71 +
72 + def sampler_output(
73 + self,
74 + seq_group_metadata_list: List[SequenceGroupMetadata],
75 + blocks_to_swap_in: Dict[int, int],
76 + blocks_to_swap_out: Dict[int, int],
77 + blocks_to_copy: Dict[int, List[int]],
78 + sample_len: int,
79 + ) -> Tuple[Optional[List[SamplerOutput]], bool]:
80 + """NGram match algo to pick proposal candidate. Returns the list of
81 + sampler output, one per SequenceGroupMetadata.
82 +
83 + For ngram worker, we already done needed transposed internal, so the
84 + indicator pass to sampler_output_to_torch shall be False.
85 + """
86 + self._raise_if_unsupported(
87 + seq_group_metadata_list,
88 + blocks_to_swap_in,
89 + blocks_to_swap_out,
90 + blocks_to_copy,
91 + )
92 +
93 + arr = []
94 + has_spec_out = False
95 + for seq_group_metadata in seq_group_metadata_list:
96 + seq_data = next(iter(seq_group_metadata.seq_data.values()))
97 +
98 + input_ids = torch.as_tensor(seq_data.get_token_ids(),
99 + dtype=torch.long,
100 + device=self.device)
101 + input_length = seq_data.get_len()
102 +
103 + for ngram_size in range(
104 + min(self.ngram_prompt_lookup_max, input_length - 1),
105 + self.ngram_prompt_lookup_min,
106 + -1,
107 + ):
108 + ngram_tensor = input_ids[-1 * ngram_size:]
109 + windows = input_ids.unfold(dimension=0,
110 + size=ngram_size,
111 + step=1)
112 + matches = (windows == ngram_tensor).all(dim=1)
113 + match_indices = matches.nonzero(as_tuple=True)[0]
114 + if match_indices.size()[0] > 1:
115 + has_spec_out = True
116 + res = seq_data.get_token_ids()
117 + res = res[match_indices[0] + ngram_size:match_indices[0] +
118 + ngram_size + sample_len]
119 + res_len = len(res)
120 + # pad 0 towards output as sample_len tokens required
121 + res += [0] * (sample_len - res_len)
122 +
123 + break
124 + else:
125 + # if no candidate found, fill with 0
126 + res = [0] * sample_len
127 +
128 + arr.append(res)
129 +
130 + if not has_spec_out:
131 + return None, False
132 +
133 + outputs = []
134 + token_ids = torch.as_tensor(arr, dtype=torch.long, device=self.device)
135 + indices = token_ids.unsqueeze(2)
136 +
137 + token_probs = torch.zeros(
138 + (len(seq_group_metadata_list), sample_len, self.vocab_size),
139 + dtype=torch.float32,
140 + device=self.device,
141 + )
142 + token_probs.scatter_(2, indices, 1)
143 + for i in range(len(seq_group_metadata_list)):
144 + outputs.append(
145 + SamplerOutput(
146 + outputs=None,
147 + sampled_token_probs=token_probs[i],
148 + sampled_token_ids=token_ids[i],
149 + ))
150 + return outputs, False
151 +
152 + def get_spec_proposals(
153 + self,
154 + seq_group_metadata_list: List[SequenceGroupMetadata],
155 + blocks_to_swap_in: Dict[int, int],
156 + blocks_to_swap_out: Dict[int, int],
157 + blocks_to_copy: Dict[int, List[int]],
158 + max_proposal_len: int,
159 + ) -> SpeculativeProposals:
160 + """Produce speculations given an input batch of sequences. The number of
161 + speculative tokens per sequence is determined by max_proposal_len.
162 + """
163 +
164 + return self._proposer.get_proposals(
165 + seq_group_metadata_list,
166 + blocks_to_swap_in,
167 + blocks_to_swap_out,
168 + blocks_to_copy,
169 + max_proposal_len,
170 + )
171 +
172 + def _raise_if_unsupported(
173 + self,
174 + seq_group_metadata_list: List[SequenceGroupMetadata],
175 + blocks_to_swap_in: Dict[int, int],
176 + blocks_to_swap_out: Dict[int, int],
177 + blocks_to_copy: Dict[int, List[int]],
178 + ) -> None:
179 + """NGramWorker does not yet implement support for cache swap
180 + operations or beam search.
181 + """
182 + if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
183 + raise NotImplementedError(
184 + "NGramWorker does not support cache operations")
185 +
186 + if any(
187 + len(seq_group_metadata.seq_data.keys()) != 1
188 + for seq_group_metadata in seq_group_metadata_list):
189 + raise NotImplementedError(
190 + "NGramWorker does not support beam search.")
vllm/spec_decode/spec_decode_worker.py
+32 -13
@@ -12,6 +12,7 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
12 SpeculativeScorer, SpeculativeScores)
13 from vllm.spec_decode.metrics import AsyncMetricsCollector
14 from vllm.spec_decode.multi_step_worker import MultiStepWorker
15 +from vllm.spec_decode.ngram_worker import NGramWorker
16 from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
17 split_batch_by_proposal_len)
18 from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
@@ -48,8 +49,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
49 """
50
51 @classmethod
51 - def from_workers(cls, proposer_worker: MultiStepWorker,
52 - scorer_worker: WorkerBase) -> "SpecDecodeWorker":
52 + def create_worker(
53 + cls,
54 + scorer_worker: WorkerBase,
55 + draft_worker_kwargs,
56 + ) -> "SpecDecodeWorker":
57 +
58 + if "ngram_prompt_lookup_max" in draft_worker_kwargs:
59 + ngram_prompt_lookup_max = (
60 + draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
61 + ngram_prompt_lookup_min = (
62 + draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
63 + else:
64 + ngram_prompt_lookup_max = 0
65 +
66 + if ngram_prompt_lookup_max > 0:
67 + proposer_worker = NGramWorker(**draft_worker_kwargs)
68 + proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
69 + ngram_prompt_lookup_max)
70 + else:
71 + proposer_worker = MultiStepWorker(**draft_worker_kwargs)
72 +
73 return SpecDecodeWorker(
74 proposer_worker,
75 scorer_worker,
@@ -59,7 +79,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
79
80 def __init__(
81 self,
62 - proposer_worker: MultiStepWorker,
82 + proposer_worker: WorkerBase,
83 scorer_worker: WorkerBase,
84 rejection_sampler: RejectionSampler,
85 metrics_collector: Optional[AsyncMetricsCollector] = None,
@@ -134,8 +154,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
154 """
155 (self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor
156 ) = True
137 - (self.proposer_worker.model_runner.model.sampler.
138 - include_gpu_probs_tensor) = True
157 + self.proposer_worker.set_include_gpu_probs_tensor()
158
159 def determine_num_available_blocks(self) -> Tuple[int, int]:
160 """Determine the number of cache blocks to use.
@@ -183,8 +202,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
202 "speculative decoding "
203 "requires non-None seq_group_metadata_list")
204
186 - logger.info("spec_decode_worker.execute_model num_lookahead_slots=%d",
187 - num_lookahead_slots)
205 + #logger.info("spec_decode_worker.execute_model num_lookahead_slots=%d",
206 + # num_lookahead_slots)
207
208 # If no spec tokens, call the proposer and scorer workers normally.
209 # Used for prefill.
@@ -216,7 +235,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
235 proposer and scorer model so that the KV cache is consistent between the
236 two.
237 """
219 - logger.info("run proposer worker no spec")
238 + #logger.info("run proposer worker no spec")
239
240 self.proposer_worker.execute_model(
241 seq_group_metadata_list=seq_group_metadata_list,
@@ -225,7 +244,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
244 blocks_to_copy=blocks_to_copy,
245 )
246
228 - logger.info("run target worker no spec")
247 + #logger.info("run target worker no spec")
248 sampler_output = self.scorer_worker.execute_model(
249 seq_group_metadata_list=seq_group_metadata_list,
250 blocks_to_swap_in=blocks_to_swap_in,
@@ -259,7 +278,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
278 sequence.
279 """
280
262 - logger.info("get spec proposals")
281 + #logger.info("get spec proposals")
282 # Generate proposals using draft worker.
283 assert blocks_to_swap_in is not None
284 assert blocks_to_swap_out is not None
@@ -268,7 +287,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
287 seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
288 blocks_to_copy, k)
289
271 - logger.info("score proposals")
290 + #logger.info("score proposals")
291 proposal_scores = self.scorer.score_proposals(
292 seq_group_metadata_list,
293 blocks_to_swap_in,
@@ -278,11 +297,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
297 proposals,
298 )
299
281 - logger.info("verify proposals")
300 + #logger.info("verify proposals")
301 accepted_token_ids = self._verify_tokens(seq_group_metadata_list,
302 proposal_scores, proposals, k)
303
285 - logger.info("create output list")
304 + #logger.info("create output list")
305 return self._create_output_sampler_list(seq_group_metadata_list,
306 accepted_token_ids, k)
307
vllm/spec_decode/top1_proposer.py new
+200
@@ -0,0 +1,200 @@
1 +from typing import Dict, List, Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6 +from vllm.spec_decode.interfaces import (SpeculativeProposals,
7 + SpeculativeProposer)
8 +from vllm.spec_decode.util import sampler_output_to_torch
9 +from vllm.worker.worker_base import WorkerBase
10 +
11 +
12 +class Top1Proposer(SpeculativeProposer):
13 + """Helper class which separates out sequences which would exceed the max
14 + model length when speculated upon.
15 +
16 + This allows combinations of models such as JackFram/llama-68m draft with
17 + meta-llama/Llama2-13b-chat-hf, as llama-68m has max_position_embeddings of
18 + 2048 while Llama2-13b has max_position_embeddings of 4096.
19 +
20 + We treat the sequences which exceed the proposal draft model length as
21 + "non-spec sequences". Essentially they skip the draft model and go through
22 + normal decoding in the target model.
23 +
24 + Currently, only proposal_lens of 0 and k are supported, where k is a global
25 + batch proposal length. In the future vLLM should support per-sequence
26 + proposal lengths.
27 + """
28 +
29 + def __init__(
30 + self,
31 + worker: WorkerBase,
32 + device: str,
33 + vocab_size: int,
34 + max_proposal_len: Optional[int] = None,
35 + ):
36 + self._worker = worker
37 + self._device = device
38 + self.max_proposal_len = max_proposal_len
39 + self._vocab_size = vocab_size
40 +
41 + def get_proposals(
42 + self,
43 + seq_group_metadata_list: List[SequenceGroupMetadata],
44 + blocks_to_swap_in: Dict[int, int],
45 + blocks_to_swap_out: Dict[int, int],
46 + blocks_to_copy: Dict[int, List[int]],
47 + proposal_len: int,
48 + ) -> SpeculativeProposals:
49 + """Get speculative proposals given the input batch.
50 +
51 + Sequences which would exceed the max model length are skipped during
52 + speculation.
53 + """
54 +
55 + # Split speculative- and non-speculative- sequences.
56 + (
57 + proposal_lens,
58 + nonzero_proposal_len_seqs,
59 + nonzero_proposal_len_indices,
60 + ) = self._split_by_max_model_len(seq_group_metadata_list, proposal_len)
61 +
62 + if nonzero_proposal_len_seqs:
63 + # Speculate tokens using the draft worker for the speculative
64 + # sequences.
65 + # If sampler_transposed is true, then maybe_sampler_output's
66 + # token_ids is like [batch] format in proposal_len size list,
67 + # while if it is false, the format would be [proposal_len]
68 + # in batch size list
69 + maybe_sampler_output, transposed = self._worker.sampler_output(
70 + seq_group_metadata_list=nonzero_proposal_len_seqs,
71 + blocks_to_swap_in=blocks_to_swap_in,
72 + blocks_to_swap_out=blocks_to_swap_out,
73 + blocks_to_copy=blocks_to_copy,
74 + sample_len=proposal_len,
75 + )
76 + else:
77 + # If no sequences can be speculated, set sampler output to None.
78 + maybe_sampler_output = None
79 + transposed = False
80 +
81 + # Combine speculative- and non-speculative sequences into the same
82 + # representation.
83 + proposal_tokens, proposal_probs, proposal_lens = self._merge_outputs(
84 + batch_size=len(seq_group_metadata_list),
85 + proposal_len=proposal_len,
86 + maybe_sampler_output=maybe_sampler_output,
87 + proposal_lens=proposal_lens,
88 + nonzero_proposal_len_indices=nonzero_proposal_len_indices,
89 + sampler_transposed=transposed,
90 + )
91 +
92 + proposals = SpeculativeProposals(
93 + proposal_token_ids=proposal_tokens,
94 + proposal_probs=proposal_probs,
95 + proposal_lens=proposal_lens,
96 + )
97 +
98 + return proposals
99 +
100 + def _split_by_max_model_len(
101 + self,
102 + seq_group_metadata_list: List[SequenceGroupMetadata],
103 + proposal_len: int,
104 + ) -> Tuple[List[int], List[SequenceGroupMetadata], List[int]]:
105 + """Determine which sequences would exceed the max model length."""
106 +
107 + proposal_lens: List[int] = []
108 + nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
109 + nonzero_proposal_len_indices: List[int] = []
110 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
111 + seq_data = next(iter(seq_group_metadata.seq_data.values()))
112 + seq_len = seq_data.get_len()
113 +
114 + # Currently only proposal lens of 0 or the global batch proposal len
115 + # are supported.
116 + # If max_proposal_len is defined, then we shall no exccess this
117 + # quota for nonzero_proposal
118 + if (self.max_proposal_len is None
119 + or seq_len + proposal_len < self.max_proposal_len):
120 + proposal_lens.append(proposal_len)
121 + nonzero_proposal_len_seqs.append(seq_group_metadata)
122 + nonzero_proposal_len_indices.append(i)
123 + else:
124 + proposal_lens.append(0)
125 +
126 + return (
127 + proposal_lens,
128 + nonzero_proposal_len_seqs,
129 + nonzero_proposal_len_indices,
130 + )
131 +
132 + def _merge_outputs(
133 + self,
134 + batch_size: int,
135 + proposal_len: int,
136 + maybe_sampler_output: Optional[SamplerOutput],
137 + proposal_lens: List[int],
138 + nonzero_proposal_len_indices: List[int],
139 + sampler_transposed: bool,
140 + ) -> Tuple[torch.Tensor, torch.tensor, torch.Tensor]:
141 + """After speculations are produced, merge the speculation results with
142 + the skipped sequences.
143 + """
144 + if maybe_sampler_output is None:
145 + # If no speculative tokens, the sampler output will be None.
146 + # In this case we return empty proposals.
147 + proposal_tokens = torch.full(
148 + size=(
149 + batch_size,
150 + proposal_len,
151 + ),
152 + fill_value=-1,
153 + dtype=torch.long,
154 + device=self._device,
155 + )
156 + proposal_probs = torch.zeros(
157 + batch_size,
158 + proposal_len,
159 + self._vocab_size,
160 + dtype=torch.float32,
161 + device=self._device,
162 + )
163 + proposal_lens_tensor = torch.zeros(len(proposal_lens),
164 + dtype=torch.long,
165 + device=self._device)
166 + return proposal_tokens, proposal_probs, proposal_lens_tensor
167 +
168 + sampler_output = maybe_sampler_output
169 + proposal_tokens, proposal_probs = sampler_output_to_torch(
170 + sampler_output, sampler_transposed)
171 +
172 + # Now, reformat the output GPU tensors such that each sequence has
173 + # a proposal. the proposal can be empty, e.g. [-1, -1, -1]
174 +
175 + entire_proposal_tokens = torch.full(
176 + size=(batch_size, *proposal_tokens.shape[1:]),
177 + fill_value=-1,
178 + dtype=torch.long,
179 + device=self._device,
180 + )
181 + entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens
182 + entire_proposal_probs = torch.zeros(
183 + batch_size,
184 + *proposal_probs.shape[1:],
185 + dtype=torch.float32,
186 + device=self._device,
187 + )
188 + entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
189 +
190 + proposal_tokens, proposal_probs = (
191 + entire_proposal_tokens,
192 + entire_proposal_probs,
193 + )
194 +
195 + proposal_lens_tensor = torch.zeros(batch_size,
196 + dtype=torch.long,
197 + device=self._device)
198 + proposal_lens_tensor[nonzero_proposal_len_indices] = proposal_len
199 +
200 + return proposal_tokens, proposal_probs, proposal_lens_tensor
vllm/spec_decode/util.py
+12 -4
@@ -49,10 +49,13 @@ def split_batch_by_proposal_len(
49
50
51 def sampler_output_to_torch(
52 - sampler_output_list: List[SamplerOutput],
53 -) -> Tuple[torch.Tensor, torch.Tensor]:
52 + sampler_output_list: List[SamplerOutput],
53 + sampler_transposed: bool) -> Tuple[torch.Tensor, torch.Tensor]:
54 """Utility function which converts a list of SamplerOutput to tensors.
55
56 + sampler_transposed here is used as the indicator for whether
57 + we need do additional tensor transpose logic here.
58 +
59 Returns:
60 sampled_token_ids: torch.Tensor
61 shape: [batch_size, len(sampler_output_list)]
@@ -68,7 +71,10 @@ def sampler_output_to_torch(
71 for sampler_output in sampler_output_list
72 ],
73 dim=0,
71 - ).transpose(0, 1)
74 + )
75 +
76 + if sampler_transposed:
77 + sampled_token_probs = sampled_token_probs.transpose(0, 1)
78
79 # shape: [batch_size, num_sampler_output]
80 sampled_token_ids = torch.stack(
@@ -77,7 +83,9 @@ def sampler_output_to_torch(
83 for sampler_output in sampler_output_list
84 ],
85 dim=0,
80 - ).transpose(0, 1)
86 + )
87 + if sampler_transposed:
88 + sampled_token_ids = sampled_token_ids.transpose(0, 1)
89
90 return sampled_token_ids, sampled_token_probs
91