A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Speculative decoding] Add ngram prompt lookup decoding (#4237)
Co-authored-by: Lei Wen <wenlei03@qiyi.com>
leiwen83 committed
May 2, 2024 at 02:13 UTC
b38e42fbca978d62cc8330bdcf8da91c72cb2ebc
14 files changed
+1004
-319
tests/spec_decode/e2e/conftest.py
+58
@@ -1,4 +1,5 @@
1
import asyncio
2
+from itertools import cycle
3
from typing import List, Optional, Tuple, Union
4
5
import pytest
@@ -185,3 +186,60 @@ def get_output_from_llm_generator(
186
del llm
187
188
return tokens, token_ids
189
+
190
+
191
+def run_greedy_equality_correctness_test(baseline_llm_generator,
192
+ test_llm_generator,
193
+ batch_size,
194
+ max_output_len,
195
+ force_output_len: bool,
196
+ print_tokens: bool = False):
197
+ """Helper method that compares the outputs of both the baseline LLM and
198
+ the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
199
+ the same when temperature is zero.
200
+ """
201
+ temperature = 0.0
202
+
203
+ prompts = [
204
+ "Hello, my name is",
205
+ "The president of the United States is",
206
+ "The capital of France is",
207
+ "The future of AI is",
208
+ "San Francisco is know for its",
209
+ "Facebook was created in 2004 by",
210
+ "Curious George is a",
211
+ "Python 3.11 brings improvements to its",
212
+ ]
213
+
214
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
215
+
216
+ # If the test requires that we generated max_output_len tokens, then set the
217
+ # sampling params to ignore eos token.
218
+ ignore_eos = force_output_len
219
+
220
+ sampling_params = SamplingParams(
221
+ max_tokens=max_output_len,
222
+ ignore_eos=ignore_eos,
223
+ temperature=temperature,
224
+ )
225
+
226
+ spec_batch_tokens, spec_batch_token_ids = get_output_from_llm_generator(
227
+ test_llm_generator, prompts, sampling_params)
228
+
229
+ (baseline_batch_tokens,
230
+ baseline_batch_token_ids) = get_output_from_llm_generator(
231
+ baseline_llm_generator, prompts, sampling_params)
232
+
233
+ assert len(baseline_batch_token_ids) == len(prompts)
234
+ assert len(spec_batch_token_ids) == len(prompts)
235
+
236
+ for i, (baseline_token_ids, baseline_tokens, spec_token_ids,
237
+ spec_tokens) in enumerate(
238
+ zip(baseline_batch_token_ids, baseline_batch_tokens,
239
+ spec_batch_token_ids, spec_batch_tokens)):
240
+ if print_tokens:
241
+ print(f'{i=} {baseline_tokens=}')
242
+ print(f'{i=} {spec_tokens=}')
243
+ print(f'{i=} {baseline_token_ids=}')
244
+ print(f'{i=} {spec_token_ids=}')
245
+ assert baseline_token_ids == spec_token_ids
tests/spec_decode/e2e/test_multistep_correctness.py
renamed
+2
-58
@@ -35,7 +35,8 @@ from transformers import AutoTokenizer
35
36
from vllm import SamplingParams
37
38
-from .conftest import get_output_from_llm_generator
38
+from .conftest import (get_output_from_llm_generator,
39
+ run_greedy_equality_correctness_test)
40
41
42
@pytest.mark.parametrize(
@@ -545,60 +546,3 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
546
batch_size,
547
max_output_len=output_len,
548
force_output_len=True)
548
-
549
-
550
-def run_greedy_equality_correctness_test(baseline_llm_generator,
551
- test_llm_generator,
552
- batch_size,
553
- max_output_len,
554
- force_output_len: bool,
555
- print_tokens: bool = False):
556
- """Helper method that compares the outputs of both the baseline LLM and
557
- the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
558
- the same when temperature is zero.
559
- """
560
- temperature = 0.0
561
-
562
- prompts = [
563
- "Hello, my name is",
564
- "The president of the United States is",
565
- "The capital of France is",
566
- "The future of AI is",
567
- "San Francisco is know for its",
568
- "Facebook was created in 2004 by",
569
- "Curious George is a",
570
- "Python 3.11 brings improvements to its",
571
- ]
572
-
573
- prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
574
-
575
- # If the test requires that we generated max_output_len tokens, then set the
576
- # sampling params to ignore eos token.
577
- ignore_eos = force_output_len
578
-
579
- sampling_params = SamplingParams(
580
- max_tokens=max_output_len,
581
- ignore_eos=ignore_eos,
582
- temperature=temperature,
583
- )
584
-
585
- spec_batch_tokens, spec_batch_token_ids = get_output_from_llm_generator(
586
- test_llm_generator, prompts, sampling_params)
587
-
588
- (baseline_batch_tokens,
589
- baseline_batch_token_ids) = get_output_from_llm_generator(
590
- baseline_llm_generator, prompts, sampling_params)
591
-
592
- assert len(baseline_batch_token_ids) == len(prompts)
593
- assert len(spec_batch_token_ids) == len(prompts)
594
-
595
- for i, (baseline_token_ids, baseline_tokens, spec_token_ids,
596
- spec_tokens) in enumerate(
597
- zip(baseline_batch_token_ids, baseline_batch_tokens,
598
- spec_batch_token_ids, spec_batch_tokens)):
599
- if print_tokens:
600
- print(f'{i=} {baseline_tokens=}')
601
- print(f'{i=} {spec_tokens=}')
602
- print(f'{i=} {baseline_token_ids=}')
603
- print(f'{i=} {spec_token_ids=}')
604
- assert baseline_token_ids == spec_token_ids
tests/spec_decode/e2e/test_ngram_correctness.py
new
+172
@@ -0,0 +1,172 @@
1
+"""This docstring details important information on the testing methodology.
2
+
3
+Most of the tests rely on "greedy equality", where we expect the output of
4
+speculative decoding on a sequence to exactly match the output of normal non-
5
+speculative decoding.
6
+
7
+Since speculative decoding with rejection sampling guarantees that the output
8
+distribution matches the target model's output distribution (up to hardware
9
+numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
10
+equality.
11
+
12
+For ngram lookup, its idea comes from https://github.com/apoorvumang/prompt-lookup-decoding,
13
+and is merged into transform code base: https://github.com/huggingface/transformers/pull/27775.
14
+Since there is no model is needed for generate the proposal, we could make
15
+the testcase much simpler than drafter multi-step one.
16
+
17
+However, we still need to verify below scenario could be passed:
18
+ * Batch size 1 greedy equality
19
+ * Batch size >1 greedy equality
20
+ * Test greedy equality under preemption
21
+ * Test greedy equality under various ngram sizes / speculative sizes
22
+
23
+With those tests, we can say at least, ngram spec would not break the correctess
24
+for the target model outputs.
25
+"""
26
+
27
+import pytest
28
+
29
+from .conftest import run_greedy_equality_correctness_test
30
+
31
+
32
+@pytest.mark.parametrize(
33
+ "common_llm_kwargs",
34
+ [{
35
+ # Skip cuda graph recording for fast test.
36
+ "enforce_eager": True,
37
+
38
+ # Required for spec decode.
39
+ "use_v2_block_manager": True,
40
+
41
+ # Print spec metrics.
42
+ "disable_log_stats": False,
43
+ }])
44
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [
45
+ {
46
+ "model": "JackFram/llama-68m",
47
+ },
48
+])
49
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
50
+@pytest.mark.parametrize("test_llm_kwargs", [
51
+ {
52
+ "speculative_model": "[ngram]",
53
+ "num_speculative_tokens": 5,
54
+ "ngram_prompt_lookup_max": 3,
55
+ },
56
+])
57
+@pytest.mark.parametrize("output_len", [
58
+ 256,
59
+])
60
+@pytest.mark.parametrize("batch_size", [1, 64])
61
+@pytest.mark.parametrize("seed", [1])
62
+def test_ngram_e2e_greedy_correctness(baseline_llm_generator,
63
+ test_llm_generator, batch_size: int,
64
+ output_len: int):
65
+ """Verify greedy equality on a tiny model with different batch size."""
66
+ run_greedy_equality_correctness_test(baseline_llm_generator,
67
+ test_llm_generator,
68
+ batch_size,
69
+ max_output_len=output_len,
70
+ force_output_len=True)
71
+
72
+
73
+@pytest.mark.parametrize(
74
+ "common_llm_kwargs",
75
+ [{
76
+ "block_size": 8,
77
+ # 2 for small prompt, 256//8 for generated.
78
+ "num_gpu_blocks_override": 2 + 256 // 8,
79
+ "max_model_len": (2 + 256 // 8) * 8,
80
+
81
+ # Skip cuda graph recording for fast test.
82
+ "enforce_eager": True,
83
+
84
+ # Required for spec decode.
85
+ "use_v2_block_manager": True
86
+ }])
87
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [
88
+ {
89
+ "model": "JackFram/llama-160m",
90
+ },
91
+])
92
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
93
+@pytest.mark.parametrize("test_llm_kwargs", [
94
+ {
95
+ "speculative_model": "[ngram]",
96
+ "num_speculative_tokens": 5,
97
+ "ngram_prompt_lookup_max": 3,
98
+ },
99
+])
100
+@pytest.mark.parametrize(
101
+ "output_len",
102
+ [
103
+ # Use small output len for fast test.
104
+ 256,
105
+ ])
106
+@pytest.mark.parametrize("batch_size", [4])
107
+@pytest.mark.parametrize("seed", [1])
108
+def test_ngram_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
109
+ test_llm_generator,
110
+ batch_size: int,
111
+ output_len: int):
112
+ """Verify greedy equality, even when some sequences are preempted mid-
113
+ generation.
114
+ """
115
+ run_greedy_equality_correctness_test(baseline_llm_generator,
116
+ test_llm_generator,
117
+ batch_size,
118
+ max_output_len=output_len,
119
+ force_output_len=True)
120
+
121
+
122
+@pytest.mark.parametrize(
123
+ "common_llm_kwargs",
124
+ [{
125
+ "model": "JackFram/llama-68m",
126
+
127
+ # Skip cuda graph recording for fast test.
128
+ "enforce_eager": True,
129
+
130
+ # Required for spec decode.
131
+ "use_v2_block_manager": True
132
+ }])
133
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
134
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
135
+@pytest.mark.parametrize(
136
+ "test_llm_kwargs",
137
+ [
138
+ {
139
+ "speculative_model": "[ngram]",
140
+ "num_speculative_tokens": k,
141
+ "ngram_prompt_lookup_max": 3,
142
+ }
143
+ # Try a range of common k, as well as large speculation.
144
+ for k in [1, 3, 5]
145
+ ] + [
146
+ {
147
+ "speculative_model": "[ngram]",
148
+ "num_speculative_tokens": k,
149
+ "ngram_prompt_lookup_max": 1,
150
+ }
151
+ # Try a range of common k, as well as large speculation.
152
+ for k in [1, 3, 5]
153
+ ])
154
+@pytest.mark.parametrize("batch_size", [2])
155
+@pytest.mark.parametrize(
156
+ "output_len",
157
+ [
158
+ # Use smaller output len for fast test.
159
+ 32,
160
+ ])
161
+@pytest.mark.parametrize("seed", [1])
162
+def test_ngram_different_k(baseline_llm_generator, test_llm_generator,
163
+ batch_size: int, output_len: int):
164
+ """Verify that ngram speculative decoding produces exact equality
165
+ to without spec decode with many different values of k and
166
+ different ngram_prompt_lookup_max.
167
+ """
168
+ run_greedy_equality_correctness_test(baseline_llm_generator,
169
+ test_llm_generator,
170
+ batch_size,
171
+ max_output_len=output_len,
172
+ force_output_len=True)
tests/spec_decode/test_multi_step_worker.py
+25
-25
@@ -6,8 +6,8 @@ import torch
6
7
from vllm.model_executor.utils import set_random_seed
8
from vllm.sequence import SamplerOutput
9
-from vllm.spec_decode.multi_step_worker import (DraftModelTop1Proposer,
10
- MultiStepWorker)
9
+from vllm.spec_decode.multi_step_worker import MultiStepWorker
10
+from vllm.spec_decode.top1_proposer import Top1Proposer
11
from vllm.worker.worker import Worker
12
13
from .utils import (assert_logprobs_dict_allclose, create_batch,
@@ -117,8 +117,8 @@ def test_same_output_for_single_step():
117
118
zero_kv_cache(multi_step_worker.cache_engine)
119
set_random_seed(seed)
120
- actual_output = multi_step_worker.execute_model_multi_step(
121
- **multi_step_execute_model_data.to_dict(), num_steps=num_steps)
120
+ actual_output, _ = multi_step_worker.sampler_output(
121
+ **multi_step_execute_model_data.to_dict(), sample_len=num_steps)
122
assert len(actual_output) == num_steps
123
actual_output = actual_output[0]
124
@@ -200,8 +200,8 @@ def test_same_output_for_multi_step():
200
# Run multi-step.
201
zero_kv_cache(multi_step_worker.cache_engine)
202
set_random_seed(seed)
203
- multi_step_output = multi_step_worker.execute_model_multi_step(
204
- **execute_model_data.to_dict(), num_steps=num_steps)
203
+ multi_step_output, _ = multi_step_worker.sampler_output(
204
+ **execute_model_data.to_dict(), sample_len=num_steps)
205
206
# Run single-step repeatedly.
207
zero_kv_cache(worker.cache_engine)
@@ -266,7 +266,7 @@ def test_same_output_for_multi_step():
266
267
@torch.inference_mode()
268
def test_draft_proposals_full_speculation_len():
269
- """Verify DraftModelTop1Proposer correctly handles case where all sequences
269
+ """Verify Top1Proposer correctly handles case where all sequences
270
can speculate.
271
"""
272
k = 10
@@ -275,13 +275,13 @@ def test_draft_proposals_full_speculation_len():
275
device = 'cuda:0'
276
277
draft_worker = MagicMock()
278
- proposer = DraftModelTop1Proposer(
279
- draft_worker=draft_worker,
278
+ proposer = Top1Proposer(
279
+ worker=draft_worker,
280
device=device,
281
- max_model_len=2048,
281
vocab_size=vocab_size,
282
+ max_proposal_len=2048,
283
)
284
- draft_worker.execute_model_multi_step.return_value = [
284
+ draft_worker.sampler_output.return_value = [
285
SamplerOutput(
286
outputs=[],
287
sampled_token_probs=torch.rand(batch_size,
@@ -294,13 +294,13 @@ def test_draft_proposals_full_speculation_len():
294
device=device,
295
dtype=torch.long),
296
) for _ in range(k)
297
- ]
297
+ ], True
298
299
execute_model_data, _, _ = create_batch(batch_size, k)
300
301
proposals = proposer.get_proposals(
302
**execute_model_data.to_dict(),
303
- max_proposal_len=k,
303
+ proposal_len=k,
304
)
305
306
assert torch.is_tensor(proposals.proposal_token_ids)
@@ -315,7 +315,7 @@ def test_draft_proposals_full_speculation_len():
315
316
@torch.inference_mode()
317
def test_draft_proposals_no_speculations():
318
- """Verify DraftModelTop1Proposer correctly handles case where no sequences
318
+ """Verify Top1Proposer correctly handles case where no sequences
319
can speculate.
320
"""
321
k = 10
@@ -325,11 +325,11 @@ def test_draft_proposals_no_speculations():
325
prompt_len = 10
326
327
draft_worker = MagicMock()
328
- proposer = DraftModelTop1Proposer(
329
- draft_worker=draft_worker,
328
+ proposer = Top1Proposer(
329
+ worker=draft_worker,
330
device=device,
331
- max_model_len=prompt_len + k - 1,
331
vocab_size=vocab_size,
332
+ max_proposal_len=prompt_len + k - 1,
333
)
334
335
execute_model_data, _, _ = create_batch(batch_size,
@@ -338,7 +338,7 @@ def test_draft_proposals_no_speculations():
338
339
proposals = proposer.get_proposals(
340
**execute_model_data.to_dict(),
341
- max_proposal_len=k,
341
+ proposal_len=k,
342
)
343
344
assert torch.is_tensor(proposals.proposal_token_ids)
@@ -353,7 +353,7 @@ def test_draft_proposals_no_speculations():
353
354
@torch.inference_mode()
355
def test_draft_proposals_mixed_k():
356
- """Verify DraftModelTop1Proposer correctly handles case some sequences can
356
+ """Verify Top1Proposer correctly handles case some sequences can
357
speculate and some can't.
358
"""
359
k = 10
@@ -374,14 +374,14 @@ def test_draft_proposals_mixed_k():
374
for _ in range(expected_num_no_proposal_seqs)] + [small_prompt_len]
375
376
draft_worker = MagicMock()
377
- proposer = DraftModelTop1Proposer(
378
- draft_worker=draft_worker,
377
+ proposer = Top1Proposer(
378
+ worker=draft_worker,
379
device=device,
380
- max_model_len=long_prompt_len + prev_output_token_len + k - 1,
380
vocab_size=vocab_size,
381
+ max_proposal_len=long_prompt_len + prev_output_token_len + k - 1,
382
)
383
384
- draft_worker.execute_model_multi_step.return_value = [
384
+ draft_worker.sampler_output.return_value = [
385
SamplerOutput(
386
outputs=[],
387
sampled_token_probs=torch.rand(expected_num_proposal_seqs,
@@ -395,7 +395,7 @@ def test_draft_proposals_mixed_k():
395
device=device,
396
dtype=torch.long),
397
) for _ in range(k)
398
- ]
398
+ ], True
399
400
execute_model_data, _, _ = create_batch(
401
batch_size,
@@ -406,7 +406,7 @@ def test_draft_proposals_mixed_k():
406
407
proposals = proposer.get_proposals(
408
**execute_model_data.to_dict(),
409
- max_proposal_len=k,
409
+ proposal_len=k,
410
)
411
412
assert torch.is_tensor(proposals.proposal_token_ids)
tests/spec_decode/test_ngram_worker.py
new
+206
@@ -0,0 +1,206 @@
1
+import torch
2
+
3
+from vllm.spec_decode.ngram_worker import NGramWorker
4
+from vllm.spec_decode.top1_proposer import Top1Proposer
5
+
6
+from .utils import (create_execute_model_data,
7
+ create_seq_group_metadata_from_prompts, create_worker)
8
+
9
+
10
+def test_ngram_algo_correctness_for_single_no_match():
11
+ """Verify our ngram algo find the right candidate in the prompt
12
+
13
+ For the scenario cannot find any candidate in one single batch
14
+ """
15
+ block_size = 32
16
+ num_gpu_blocks = 2048 // block_size
17
+ seed = 100
18
+ model_name = 'JackFram/llama-68m'
19
+ vocab_size = 32_000
20
+ device = 'cuda:0'
21
+
22
+ ngram_worker = create_worker(
23
+ NGramWorker,
24
+ model_name,
25
+ block_size,
26
+ num_gpu_blocks,
27
+ seed,
28
+ )
29
+
30
+ proposer = Top1Proposer(
31
+ worker=ngram_worker,
32
+ device=device,
33
+ vocab_size=vocab_size,
34
+ max_proposal_len=20,
35
+ )
36
+
37
+ # set ngram window (0, 3], which is window=1/2/3
38
+ ngram_worker.set_ngram_window_size(0, 3)
39
+
40
+ prompts = [
41
+ # shall find no candidate
42
+ [1, 2, 3, 4, 5, 6, 7],
43
+ ]
44
+
45
+ proposal_len = 5
46
+ final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
47
+ ngram_sampler_output_data = create_execute_model_data(
48
+ seq_group_metadata_list=create_seq_group_metadata_from_prompts(
49
+ prompts, num_gpu_blocks, block_size,
50
+ final_seq_lens=final_seq_lens))
51
+
52
+ proposals = proposer.get_proposals(
53
+ **ngram_sampler_output_data.to_dict(),
54
+ proposal_len=proposal_len,
55
+ )
56
+
57
+ assert torch.is_tensor(proposals.proposal_token_ids)
58
+ assert torch.is_tensor(proposals.proposal_probs)
59
+
60
+ assert proposals.proposal_token_ids.shape == torch.Size([1, proposal_len])
61
+ assert proposals.proposal_probs.shape[:-1] == torch.Size([1, proposal_len])
62
+ assert proposals.proposal_lens.shape == torch.Size([1])
63
+ assert proposals.proposal_lens.tolist() == [0]
64
+
65
+
66
+def test_ngram_algo_correctness_for_batches_not_match_all():
67
+ """Verify our ngram algo find the right candidate in the prompt
68
+
69
+ For the scenario find some candidate not full in batchs
70
+ """
71
+ block_size = 32
72
+ num_gpu_blocks = 2048 // block_size
73
+ seed = 100
74
+ model_name = 'JackFram/llama-68m'
75
+ vocab_size = 32_000
76
+ device = 'cuda:0'
77
+
78
+ ngram_worker = create_worker(
79
+ NGramWorker,
80
+ model_name,
81
+ block_size,
82
+ num_gpu_blocks,
83
+ seed,
84
+ )
85
+
86
+ proposer = Top1Proposer(
87
+ worker=ngram_worker,
88
+ device=device,
89
+ vocab_size=vocab_size,
90
+ max_proposal_len=20,
91
+ )
92
+
93
+ # set ngram window (0, 3], which is window=1/2/3
94
+ ngram_worker.set_ngram_window_size(0, 3)
95
+
96
+ prompts = [
97
+ # shall find no candidate
98
+ [1, 2, 3, 4, 5, 6, 7],
99
+ # shall find candidate 12,13,14,15,16
100
+ [11, 12, 13, 14, 15, 16, 11],
101
+ # shall find candidate 23,24,25,26,21
102
+ [21, 21, 22, 23, 24, 25, 26, 21, 22],
103
+ # shall find candidate 34,35,36,37,38
104
+ [31, 32, 31, 32, 33, 34, 35, 36, 37, 38, 31, 32, 33],
105
+ # shall find no candidate as exceed max_proposal_len
106
+ [
107
+ 31, 32, 31, 32, 31, 32, 31, 32, 31, 32, 31, 32, 33, 34, 35, 36, 37,
108
+ 38, 31, 32, 33
109
+ ],
110
+ ]
111
+
112
+ proposal_len = 5
113
+ final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
114
+ ngram_sampler_output_data = create_execute_model_data(
115
+ seq_group_metadata_list=create_seq_group_metadata_from_prompts(
116
+ prompts, num_gpu_blocks, block_size,
117
+ final_seq_lens=final_seq_lens))
118
+
119
+ proposals = proposer.get_proposals(
120
+ **ngram_sampler_output_data.to_dict(),
121
+ proposal_len=proposal_len,
122
+ )
123
+
124
+ assert torch.is_tensor(proposals.proposal_token_ids)
125
+ assert torch.is_tensor(proposals.proposal_probs)
126
+
127
+ assert proposals.proposal_token_ids.shape == torch.Size([5, proposal_len])
128
+ assert proposals.proposal_probs.shape[:-1] == torch.Size([5, proposal_len])
129
+ assert proposals.proposal_lens.shape == torch.Size([5])
130
+
131
+ assert proposals.proposal_lens.tolist(
132
+ ) == [proposal_len for _ in range(4)] + [0]
133
+
134
+ for i in range(proposal_len):
135
+ assert proposals.proposal_token_ids[0][i] == 0
136
+ assert proposals.proposal_token_ids[1][i] == prompts[1][i + 1]
137
+ assert proposals.proposal_token_ids[2][i] == prompts[2][i + 3]
138
+ assert proposals.proposal_token_ids[3][i] == prompts[3][i + 5]
139
+ assert proposals.proposal_token_ids[4][i] == -1
140
+
141
+
142
+def test_ngram_algo_correctness_for_batches_match_all():
143
+ """Verify our ngram algo find the right candidate in the prompt
144
+
145
+ For the scenario find candidate in all batchs
146
+ """
147
+
148
+ block_size = 32
149
+ num_gpu_blocks = 2048 // block_size
150
+ seed = 100
151
+ model_name = 'JackFram/llama-68m'
152
+ vocab_size = 32_000
153
+ device = 'cuda:0'
154
+
155
+ ngram_worker = create_worker(
156
+ NGramWorker,
157
+ model_name,
158
+ block_size,
159
+ num_gpu_blocks,
160
+ seed,
161
+ )
162
+
163
+ proposer = Top1Proposer(
164
+ worker=ngram_worker,
165
+ device=device,
166
+ vocab_size=vocab_size,
167
+ max_proposal_len=20,
168
+ )
169
+
170
+ # set ngram window (0, 3], which is window=1/2/3
171
+ ngram_worker.set_ngram_window_size(0, 3)
172
+
173
+ prompts = [
174
+ # shall find candidate 12,13,14,15,16
175
+ [11, 12, 13, 14, 15, 16, 11],
176
+ # shall find candidate 23,24,25,26,21
177
+ [21, 21, 22, 23, 24, 25, 26, 21, 22],
178
+ # shall find candidate 34,35,36,37,38
179
+ [31, 32, 31, 32, 33, 34, 35, 36, 37, 38, 31, 32, 33],
180
+ ]
181
+
182
+ proposal_len = 5
183
+ final_seq_lens = [len(prompt) + proposal_len for prompt in prompts]
184
+ ngram_sampler_output_data = create_execute_model_data(
185
+ seq_group_metadata_list=create_seq_group_metadata_from_prompts(
186
+ prompts, num_gpu_blocks, block_size,
187
+ final_seq_lens=final_seq_lens))
188
+
189
+ proposals = proposer.get_proposals(
190
+ **ngram_sampler_output_data.to_dict(),
191
+ proposal_len=proposal_len,
192
+ )
193
+
194
+ assert torch.is_tensor(proposals.proposal_token_ids)
195
+ assert torch.is_tensor(proposals.proposal_probs)
196
+
197
+ assert proposals.proposal_token_ids.shape == torch.Size([3, proposal_len])
198
+ assert proposals.proposal_probs.shape[:-1] == torch.Size([3, proposal_len])
199
+ assert proposals.proposal_lens.shape == torch.Size([3])
200
+
201
+ assert proposals.proposal_lens.tolist() == [proposal_len for _ in range(3)]
202
+
203
+ for i in range(proposal_len):
204
+ assert proposals.proposal_token_ids[0][i] == prompts[0][i + 1]
205
+ assert proposals.proposal_token_ids[1][i] == prompts[1][i + 3]
206
+ assert proposals.proposal_token_ids[2][i] == prompts[2][i + 5]
vllm/config.py
+59
-28
@@ -682,6 +682,8 @@ class SpeculativeConfig:
682
speculative_max_model_len: Optional[int],
683
enable_chunked_prefill: bool,
684
use_v2_block_manager: bool,
685
+ ngram_prompt_lookup_max: Optional[int],
686
+ ngram_prompt_lookup_min: Optional[int],
687
) -> Optional["SpeculativeConfig"]:
688
"""Create a SpeculativeConfig if possible, else return None.
689
@@ -708,6 +710,10 @@ class SpeculativeConfig:
710
use_v2_block_manager (bool): Whether vLLM is configured to use the
711
v2 block manager or not. Used for raising an error since the v2
712
block manager is required with spec decode.
713
+ ngram_prompt_lookup_max (Optional[int]): Max size of ngram token
714
+ window, if provided.
715
+ ngram_prompt_lookup_min (Optional[int]): Min size of ngram token
716
+ window, if provided.
717
718
Returns:
719
Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
@@ -742,39 +748,57 @@ class SpeculativeConfig:
748
draft_code_revision = None
749
draft_quantization = None
750
745
- draft_model_config = ModelConfig(
746
- model=speculative_model,
747
- tokenizer=target_model_config.tokenizer,
748
- tokenizer_mode=target_model_config.tokenizer_mode,
749
- trust_remote_code=target_model_config.trust_remote_code,
750
- dtype=target_model_config.dtype,
751
- seed=target_model_config.seed,
752
- revision=draft_revision,
753
- code_revision=draft_code_revision,
754
- tokenizer_revision=target_model_config.tokenizer_revision,
755
- max_model_len=None,
756
- quantization=draft_quantization,
757
- enforce_eager=target_model_config.enforce_eager,
758
- max_context_len_to_capture=target_model_config.
759
- max_context_len_to_capture,
760
- max_logprobs=target_model_config.max_logprobs,
761
- )
762
-
763
- draft_model_config.max_model_len = (
764
- SpeculativeConfig._maybe_override_draft_max_model_len(
765
- speculative_max_model_len,
766
- draft_model_config.max_model_len,
767
- target_model_config.max_model_len,
768
- ))
751
+ if speculative_model == "[ngram]":
752
+ assert (ngram_prompt_lookup_max is not None
753
+ and ngram_prompt_lookup_max > 0)
754
+ if ngram_prompt_lookup_min is None:
755
+ ngram_prompt_lookup_min = 0
756
+ else:
757
+ assert ngram_prompt_lookup_max > ngram_prompt_lookup_min
758
770
- draft_parallel_config = (
771
- SpeculativeConfig.create_draft_parallel_config(
772
- target_parallel_config))
759
+ # TODO: current we still need extract vocab_size from target model
760
+ # config, in future, we may try refactor it out, and set
761
+ # draft related config as None here.
762
+ draft_model_config = target_model_config
763
+ draft_parallel_config = target_parallel_config
764
+ else:
765
+ ngram_prompt_lookup_max = 0
766
+ ngram_prompt_lookup_min = 0
767
+ draft_model_config = ModelConfig(
768
+ model=speculative_model,
769
+ tokenizer=target_model_config.tokenizer,
770
+ tokenizer_mode=target_model_config.tokenizer_mode,
771
+ trust_remote_code=target_model_config.trust_remote_code,
772
+ dtype=target_model_config.dtype,
773
+ seed=target_model_config.seed,
774
+ revision=draft_revision,
775
+ code_revision=draft_code_revision,
776
+ tokenizer_revision=target_model_config.tokenizer_revision,
777
+ max_model_len=None,
778
+ quantization=draft_quantization,
779
+ enforce_eager=target_model_config.enforce_eager,
780
+ max_context_len_to_capture=target_model_config.
781
+ max_context_len_to_capture,
782
+ max_logprobs=target_model_config.max_logprobs,
783
+ )
784
+
785
+ draft_model_config.max_model_len = (
786
+ SpeculativeConfig._maybe_override_draft_max_model_len(
787
+ speculative_max_model_len,
788
+ draft_model_config.max_model_len,
789
+ target_model_config.max_model_len,
790
+ ))
791
+
792
+ draft_parallel_config = (
793
+ SpeculativeConfig.create_draft_parallel_config(
794
+ target_parallel_config))
795
796
return SpeculativeConfig(
797
draft_model_config,
798
draft_parallel_config,
799
num_speculative_tokens,
800
+ ngram_prompt_lookup_max,
801
+ ngram_prompt_lookup_min,
802
)
803
804
@staticmethod
@@ -842,6 +866,8 @@ class SpeculativeConfig:
866
draft_model_config: ModelConfig,
867
draft_parallel_config: ParallelConfig,
868
num_speculative_tokens: int,
869
+ ngram_prompt_lookup_max: int,
870
+ ngram_prompt_lookup_min: int,
871
):
872
"""Create a SpeculativeConfig object.
873
@@ -854,6 +880,8 @@ class SpeculativeConfig:
880
self.draft_model_config = draft_model_config
881
self.draft_parallel_config = draft_parallel_config
882
self.num_speculative_tokens = num_speculative_tokens
883
+ self.ngram_prompt_lookup_max = ngram_prompt_lookup_max
884
+ self.ngram_prompt_lookup_min = ngram_prompt_lookup_min
885
886
self._verify_args()
887
@@ -877,7 +905,10 @@ class SpeculativeConfig:
905
return self.num_speculative_tokens
906
907
def __repr__(self) -> str:
880
- draft_model = self.draft_model_config.model
908
+ if self.ngram_prompt_lookup_max > 0:
909
+ draft_model = "[ngram]"
910
+ else:
911
+ draft_model = self.draft_model_config.model
912
num_spec_tokens = self.num_speculative_tokens
913
return f"SpeculativeConfig({draft_model=}, {num_spec_tokens=})"
914
vllm/engine/arg_utils.py
+18
@@ -75,6 +75,8 @@ class EngineArgs:
75
speculative_model: Optional[str] = None
76
num_speculative_tokens: Optional[int] = None
77
speculative_max_model_len: Optional[int] = None
78
+ ngram_prompt_lookup_max: Optional[int] = None
79
+ ngram_prompt_lookup_min: Optional[int] = None
80
81
def __post_init__(self):
82
if self.tokenizer is None:
@@ -449,6 +451,20 @@ class EngineArgs:
451
'draft model. Sequences over this length will skip '
452
'speculation.')
453
454
+ parser.add_argument(
455
+ '--ngram-prompt-lookup-max',
456
+ type=int,
457
+ default=EngineArgs.ngram_prompt_lookup_max,
458
+ help='Max size of window for ngram prompt lookup in speculative '
459
+ 'decoding.')
460
+
461
+ parser.add_argument(
462
+ '--ngram-prompt-lookup-min',
463
+ type=int,
464
+ default=EngineArgs.ngram_prompt_lookup_min,
465
+ help='Min size of window for ngram prompt lookup in speculative '
466
+ 'decoding.')
467
+
468
parser.add_argument('--model-loader-extra-config',
469
type=str,
470
default=EngineArgs.model_loader_extra_config,
@@ -502,6 +518,8 @@ class EngineArgs:
518
speculative_max_model_len=self.speculative_max_model_len,
519
enable_chunked_prefill=self.enable_chunked_prefill,
520
use_v2_block_manager=self.use_v2_block_manager,
521
+ ngram_prompt_lookup_max=self.ngram_prompt_lookup_max,
522
+ ngram_prompt_lookup_min=self.ngram_prompt_lookup_min,
523
)
524
525
scheduler_config = SchedulerConfig(
vllm/executor/gpu_executor.py
+4
-4
@@ -73,7 +73,6 @@ class GPUExecutor(ExecutorBase):
73
"""
74
assert self.speculative_config is not None
75
76
- from vllm.spec_decode.multi_step_worker import MultiStepWorker
76
from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
77
78
target_worker = self._create_worker()
@@ -86,10 +85,11 @@ class GPUExecutor(ExecutorBase):
85
# TODO allow draft-model specific load config.
86
#load_config=self.load_config,
87
)
89
- draft_worker = MultiStepWorker(**draft_worker_kwargs)
88
91
- spec_decode_worker = SpecDecodeWorker.from_workers(
92
- proposer_worker=draft_worker, scorer_worker=target_worker)
89
+ spec_decode_worker = SpecDecodeWorker.create_worker(
90
+ scorer_worker=target_worker,
91
+ draft_worker_kwargs=draft_worker_kwargs,
92
+ )
93
94
assert self.parallel_config.world_size == 1, (
95
"GPUExecutor only supports single GPU.")
vllm/spec_decode/batch_expansion.py
+2
-2
@@ -333,13 +333,13 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
333
sampler_output.sampled_token_probs = spec_probs
334
sampler_output.sampled_token_ids = spec_sampled_tokens
335
target_token_ids, target_probs = sampler_output_to_torch(
336
- [sampler_output])
336
+ [sampler_output], True)
337
338
# Convert non-speculative output tokens to tensors.
339
sampler_output.sampled_token_probs = non_spec_probs
340
sampler_output.sampled_token_ids = non_spec_sampled_tokens
341
non_spec_target_token_ids, non_spec_target_probs = (
342
- sampler_output_to_torch([sampler_output]))
342
+ sampler_output_to_torch([sampler_output], True))
343
344
return (target_token_ids, target_probs, non_spec_target_token_ids,
345
non_spec_target_probs)
vllm/spec_decode/multi_step_worker.py
+24
-185
@@ -1,12 +1,11 @@
1
import copy
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import Dict, List, Tuple
3
4
import torch
5
6
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
-from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
- SpeculativeProposer)
9
-from vllm.spec_decode.util import sampler_output_to_torch
7
+from vllm.spec_decode.interfaces import SpeculativeProposals
8
+from vllm.spec_decode.top1_proposer import Top1Proposer
9
from vllm.worker.worker import Worker
10
11
@@ -26,29 +25,37 @@ class MultiStepWorker(Worker):
25
super().__init__(*args, **kwargs)
26
27
# Lazy initialization list.
29
- self._proposer: DraftModelTop1Proposer
28
+ self._proposer: Top1Proposer
29
30
def init_device(self):
31
super().init_device()
32
34
- self._proposer = DraftModelTop1Proposer(
33
+ self._proposer = Top1Proposer(
34
self,
35
self.device,
37
- self.max_model_len,
36
self.vocab_size,
37
+ max_proposal_len=self.max_model_len,
38
)
39
40
+ def set_include_gpu_probs_tensor(self):
41
+ # Need include_gpu_probs_tensor for multi_step_worker
42
+ self.model_runner.model.sampler.include_gpu_probs_tensor = True
43
+
44
@torch.inference_mode()
42
- def execute_model_multi_step(
45
+ def sampler_output(
46
self,
47
seq_group_metadata_list: List[SequenceGroupMetadata],
48
blocks_to_swap_in: Dict[int, int],
49
blocks_to_swap_out: Dict[int, int],
50
blocks_to_copy: Dict[int, List[int]],
48
- num_steps: int,
49
- ) -> List[SamplerOutput]:
50
- """Run the model forward pass num_steps times. Returns the list of
51
- sampler output, one per model forward pass.
51
+ sample_len: int,
52
+ ) -> Tuple[List[SamplerOutput], bool]:
53
+ """Run the model forward pass sample_len times. Returns the list of
54
+ sampler output, one per model forward pass, along with indicator of
55
+ whether torch tensor in sampler output need to be transposed in latter
56
+ sampler_output_to_torch logic.
57
+
58
+ For multi step worker, this indicator shall be True.
59
"""
60
self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
61
blocks_to_swap_out, blocks_to_copy)
@@ -58,12 +65,12 @@ class MultiStepWorker(Worker):
65
copied_seq_group_metadata_list = self._shallow_copy_inputs(
66
seq_group_metadata_list)
67
61
- # Assert enough KV space for num_steps tokens per sequence.
62
- self._assert_enough_kv_space(seq_group_metadata_list, num_steps)
68
+ # Assert enough KV space for sample_len tokens per sequence.
69
+ self._assert_enough_kv_space(seq_group_metadata_list, sample_len)
70
64
- # Run model num_steps times.
71
+ # Run model sample_len times.
72
model_outputs = []
66
- for _ in range(num_steps):
73
+ for _ in range(sample_len):
74
model_output = super().execute_model(
75
seq_group_metadata_list=copied_seq_group_metadata_list,
76
blocks_to_swap_in=blocks_to_swap_in,
@@ -78,7 +85,7 @@ class MultiStepWorker(Worker):
85
copied_seq_group_metadata_list)
86
model_outputs.append(model_output)
87
81
- return model_outputs
88
+ return model_outputs, True
89
90
def get_spec_proposals(
91
self,
@@ -206,171 +213,3 @@ class MultiStepWorker(Worker):
213
for seq_group_metadata in seq_group_metadata_list):
214
raise NotImplementedError(
215
"MultiStepWorker does not support beam search.")
209
-
210
-
211
-class DraftModelTop1Proposer(SpeculativeProposer):
212
- """Helper class which separates out sequences which would exceed the max
213
- model length when speculated upon.
214
-
215
- This allows combinations of models such as JackFram/llama-68m draft with
216
- meta-llama/Llama2-13b-chat-hf, as llama-68m has max_position_embeddings of
217
- 2048 while Llama2-13b has max_position_embeddings of 4096.
218
-
219
- We treat the sequences which exceed the proposal draft model length as
220
- "non-spec sequences". Essentially they skip the draft model and go through
221
- normal decoding in the target model.
222
-
223
- Currently, only proposal_lens of 0 and k are supported, where k is a global
224
- batch proposal length. In the future vLLM should support per-sequence
225
- proposal lengths.
226
- """
227
-
228
- def __init__(
229
- self,
230
- draft_worker: MultiStepWorker,
231
- device: str,
232
- max_model_len: int,
233
- vocab_size: int,
234
- ):
235
- self._draft_worker = draft_worker
236
- self._device = device
237
- self._max_model_len = max_model_len
238
- self._vocab_size = vocab_size
239
-
240
- def get_proposals(
241
- self,
242
- seq_group_metadata_list: List[SequenceGroupMetadata],
243
- blocks_to_swap_in: Dict[int, int],
244
- blocks_to_swap_out: Dict[int, int],
245
- blocks_to_copy: Dict[int, List[int]],
246
- max_proposal_len: int,
247
- ) -> SpeculativeProposals:
248
- """Get speculative proposals given the input batch.
249
-
250
- Sequences which would exceed the max model length are skipped during
251
- speculation.
252
- """
253
-
254
- # Split speculative- and non-speculative- sequences.
255
- (proposal_lens, nonzero_proposal_len_seqs,
256
- nonzero_proposal_len_indices) = self._split_by_max_model_len(
257
- seq_group_metadata_list, max_proposal_len)
258
-
259
- if nonzero_proposal_len_seqs:
260
- # Speculate tokens using the draft worker for the speculative
261
- # sequences.
262
- maybe_sampler_output = self._draft_worker.execute_model_multi_step(
263
- seq_group_metadata_list=nonzero_proposal_len_seqs,
264
- blocks_to_swap_in=blocks_to_swap_in,
265
- blocks_to_swap_out=blocks_to_swap_out,
266
- blocks_to_copy=blocks_to_copy,
267
- num_steps=max_proposal_len,
268
- )
269
- else:
270
- # If no sequences can be speculated, set sampler output to None.
271
- maybe_sampler_output = None
272
-
273
- # Combine speculative- and non-speculative sequences into the same
274
- # representation.
275
- proposal_tokens, proposal_probs, proposal_lens = self._merge_outputs(
276
- batch_size=len(seq_group_metadata_list),
277
- max_proposal_len=max_proposal_len,
278
- maybe_sampler_output=maybe_sampler_output,
279
- proposal_lens=proposal_lens,
280
- nonzero_proposal_len_indices=nonzero_proposal_len_indices,
281
- )
282
-
283
- proposals = SpeculativeProposals(
284
- proposal_token_ids=proposal_tokens,
285
- proposal_probs=proposal_probs,
286
- proposal_lens=proposal_lens,
287
- )
288
-
289
- return proposals
290
-
291
- def _split_by_max_model_len(
292
- self,
293
- seq_group_metadata_list: List[SequenceGroupMetadata],
294
- max_proposal_len: int,
295
- ) -> Tuple[List[int], List[SequenceGroupMetadata], List[int]]:
296
- """Determine which sequences would exceed the max model length.
297
- """
298
-
299
- proposal_lens: List[int] = []
300
- nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
301
- nonzero_proposal_len_indices: List[int] = []
302
- for i, seq_group_metadata in enumerate(seq_group_metadata_list):
303
- seq_data = next(iter(seq_group_metadata.seq_data.values()))
304
- seq_len = seq_data.get_len()
305
-
306
- # Currently only proposal lens of 0 or the global batch proposal len
307
- # are supported.
308
- if seq_len + max_proposal_len < self._max_model_len:
309
- proposal_lens.append(max_proposal_len)
310
- nonzero_proposal_len_seqs.append(seq_group_metadata)
311
- nonzero_proposal_len_indices.append(i)
312
- else:
313
- proposal_lens.append(0)
314
-
315
- return (proposal_lens, nonzero_proposal_len_seqs,
316
- nonzero_proposal_len_indices)
317
-
318
- def _merge_outputs(
319
- self,
320
- batch_size: int,
321
- max_proposal_len: int,
322
- maybe_sampler_output: Optional[SamplerOutput],
323
- proposal_lens: List[int],
324
- nonzero_proposal_len_indices: List[int],
325
- ) -> Tuple[torch.Tensor, torch.tensor, torch.Tensor]:
326
- """After speculations are produced, merge the speculation results with
327
- the skipped sequences.
328
- """
329
- if maybe_sampler_output is None:
330
- # If no speculative tokens, the sampler output will be None.
331
- # In this case we return empty proposals.
332
- proposal_tokens = torch.full(size=(
333
- batch_size,
334
- max_proposal_len,
335
- ),
336
- fill_value=-1,
337
- dtype=torch.long,
338
- device=self._device)
339
- proposal_probs = torch.zeros(batch_size,
340
- max_proposal_len,
341
- self._vocab_size,
342
- dtype=torch.float32,
343
- device=self._device)
344
- proposal_lens_tensor = torch.zeros(len(proposal_lens),
345
- dtype=torch.long,
346
- device=self._device)
347
- return proposal_tokens, proposal_probs, proposal_lens_tensor
348
-
349
- sampler_output = maybe_sampler_output
350
- proposal_tokens, proposal_probs = sampler_output_to_torch(
351
- sampler_output)
352
-
353
- # Now, reformat the output GPU tensors such that each sequence has
354
- # a proposal. the proposal can be empty, e.g. [-1, -1, -1]
355
-
356
- entire_proposal_tokens = torch.full(size=(batch_size,
357
- *proposal_tokens.shape[1:]),
358
- fill_value=-1,
359
- dtype=torch.long,
360
- device=self._device)
361
- entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens
362
- entire_proposal_probs = torch.zeros(batch_size,
363
- *proposal_probs.shape[1:],
364
- dtype=torch.float32,
365
- device=self._device)
366
- entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
367
-
368
- proposal_tokens, proposal_probs = (entire_proposal_tokens,
369
- entire_proposal_probs)
370
-
371
- proposal_lens_tensor = torch.zeros(batch_size,
372
- dtype=torch.long,
373
- device=self._device)
374
- proposal_lens_tensor[nonzero_proposal_len_indices] = max_proposal_len
375
-
376
- return proposal_tokens, proposal_probs, proposal_lens_tensor
vllm/spec_decode/ngram_worker.py
new
+190
@@ -0,0 +1,190 @@
1
+from typing import Dict, List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6
+from vllm.spec_decode.interfaces import SpeculativeProposals
7
+from vllm.spec_decode.top1_proposer import Top1Proposer
8
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase
9
+
10
+
11
+class NGramWorker(LoraNotSupportedWorkerBase):
12
+ """NGramWorker provides a light drafter without need for model.
13
+
14
+ Current NGramWorker only implement prompt lookup decoding,
15
+ and in future we may also do RAG type drafter and other scenerios
16
+ which don't rely on LLM model to give proposals.
17
+ """
18
+
19
+ def __init__(self, *args, **kwargs):
20
+ # Get local_rank/vocab_size from kwargs attribute
21
+ self.local_rank = kwargs["local_rank"]
22
+ self.vocab_size = kwargs["model_config"].get_vocab_size()
23
+
24
+ # Lazy initialization list.
25
+ self._proposer: Top1Proposer
26
+
27
+ def set_ngram_window_size(self, ngram_prompt_lookup_min: int,
28
+ ngram_prompt_lookup_max: int):
29
+ # Search valid candidate window between
30
+ # ngram_prompt_lookup_min/ngram_prompt_lookup_max
31
+ self.ngram_prompt_lookup_max = ngram_prompt_lookup_max
32
+ self.ngram_prompt_lookup_min = ngram_prompt_lookup_min
33
+
34
+ def init_device(self):
35
+ self.device = torch.device(f"cuda:{self.local_rank}")
36
+ self.load_model = lambda *args, **kwargs: None
37
+
38
+ # Current only support Top1Proposer
39
+ self._proposer = Top1Proposer(
40
+ self,
41
+ device=self.device,
42
+ vocab_size=self.vocab_size,
43
+ )
44
+
45
+ def set_include_gpu_probs_tensor(self):
46
+ # NGram don't need gpu sampler
47
+ pass
48
+
49
+ def execute_model(
50
+ self,
51
+ seq_group_metadata_list: List[SequenceGroupMetadata],
52
+ blocks_to_swap_in: Optional[Dict[int, int]],
53
+ blocks_to_swap_out: Optional[Dict[int, int]],
54
+ blocks_to_copy: Optional[Dict[int, List[int]]],
55
+ ) -> None:
56
+ """NGram doesn't depend on model execution, just pass this function"""
57
+ pass
58
+
59
+ def determine_num_available_blocks(self) -> None:
60
+ """NGram doesn't depend on model execution, no need to check blocks"""
61
+ pass
62
+
63
+ def initialize_cache(self, num_gpu_blocks: int,
64
+ num_cpu_blocks: int) -> None:
65
+ """As there is no cache need to handle, just pass this function"""
66
+ pass
67
+
68
+ def get_cache_block_size_bytes(self):
69
+ """Return the size of a cache block in bytes."""
70
+ return 0
71
+
72
+ def sampler_output(
73
+ self,
74
+ seq_group_metadata_list: List[SequenceGroupMetadata],
75
+ blocks_to_swap_in: Dict[int, int],
76
+ blocks_to_swap_out: Dict[int, int],
77
+ blocks_to_copy: Dict[int, List[int]],
78
+ sample_len: int,
79
+ ) -> Tuple[Optional[List[SamplerOutput]], bool]:
80
+ """NGram match algo to pick proposal candidate. Returns the list of
81
+ sampler output, one per SequenceGroupMetadata.
82
+
83
+ For ngram worker, we already done needed transposed internal, so the
84
+ indicator pass to sampler_output_to_torch shall be False.
85
+ """
86
+ self._raise_if_unsupported(
87
+ seq_group_metadata_list,
88
+ blocks_to_swap_in,
89
+ blocks_to_swap_out,
90
+ blocks_to_copy,
91
+ )
92
+
93
+ arr = []
94
+ has_spec_out = False
95
+ for seq_group_metadata in seq_group_metadata_list:
96
+ seq_data = next(iter(seq_group_metadata.seq_data.values()))
97
+
98
+ input_ids = torch.as_tensor(seq_data.get_token_ids(),
99
+ dtype=torch.long,
100
+ device=self.device)
101
+ input_length = seq_data.get_len()
102
+
103
+ for ngram_size in range(
104
+ min(self.ngram_prompt_lookup_max, input_length - 1),
105
+ self.ngram_prompt_lookup_min,
106
+ -1,
107
+ ):
108
+ ngram_tensor = input_ids[-1 * ngram_size:]
109
+ windows = input_ids.unfold(dimension=0,
110
+ size=ngram_size,
111
+ step=1)
112
+ matches = (windows == ngram_tensor).all(dim=1)
113
+ match_indices = matches.nonzero(as_tuple=True)[0]
114
+ if match_indices.size()[0] > 1:
115
+ has_spec_out = True
116
+ res = seq_data.get_token_ids()
117
+ res = res[match_indices[0] + ngram_size:match_indices[0] +
118
+ ngram_size + sample_len]
119
+ res_len = len(res)
120
+ # pad 0 towards output as sample_len tokens required
121
+ res += [0] * (sample_len - res_len)
122
+
123
+ break
124
+ else:
125
+ # if no candidate found, fill with 0
126
+ res = [0] * sample_len
127
+
128
+ arr.append(res)
129
+
130
+ if not has_spec_out:
131
+ return None, False
132
+
133
+ outputs = []
134
+ token_ids = torch.as_tensor(arr, dtype=torch.long, device=self.device)
135
+ indices = token_ids.unsqueeze(2)
136
+
137
+ token_probs = torch.zeros(
138
+ (len(seq_group_metadata_list), sample_len, self.vocab_size),
139
+ dtype=torch.float32,
140
+ device=self.device,
141
+ )
142
+ token_probs.scatter_(2, indices, 1)
143
+ for i in range(len(seq_group_metadata_list)):
144
+ outputs.append(
145
+ SamplerOutput(
146
+ outputs=None,
147
+ sampled_token_probs=token_probs[i],
148
+ sampled_token_ids=token_ids[i],
149
+ ))
150
+ return outputs, False
151
+
152
+ def get_spec_proposals(
153
+ self,
154
+ seq_group_metadata_list: List[SequenceGroupMetadata],
155
+ blocks_to_swap_in: Dict[int, int],
156
+ blocks_to_swap_out: Dict[int, int],
157
+ blocks_to_copy: Dict[int, List[int]],
158
+ max_proposal_len: int,
159
+ ) -> SpeculativeProposals:
160
+ """Produce speculations given an input batch of sequences. The number of
161
+ speculative tokens per sequence is determined by max_proposal_len.
162
+ """
163
+
164
+ return self._proposer.get_proposals(
165
+ seq_group_metadata_list,
166
+ blocks_to_swap_in,
167
+ blocks_to_swap_out,
168
+ blocks_to_copy,
169
+ max_proposal_len,
170
+ )
171
+
172
+ def _raise_if_unsupported(
173
+ self,
174
+ seq_group_metadata_list: List[SequenceGroupMetadata],
175
+ blocks_to_swap_in: Dict[int, int],
176
+ blocks_to_swap_out: Dict[int, int],
177
+ blocks_to_copy: Dict[int, List[int]],
178
+ ) -> None:
179
+ """NGramWorker does not yet implement support for cache swap
180
+ operations or beam search.
181
+ """
182
+ if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
183
+ raise NotImplementedError(
184
+ "NGramWorker does not support cache operations")
185
+
186
+ if any(
187
+ len(seq_group_metadata.seq_data.keys()) != 1
188
+ for seq_group_metadata in seq_group_metadata_list):
189
+ raise NotImplementedError(
190
+ "NGramWorker does not support beam search.")
vllm/spec_decode/spec_decode_worker.py
+32
-13
@@ -12,6 +12,7 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
12
SpeculativeScorer, SpeculativeScores)
13
from vllm.spec_decode.metrics import AsyncMetricsCollector
14
from vllm.spec_decode.multi_step_worker import MultiStepWorker
15
+from vllm.spec_decode.ngram_worker import NGramWorker
16
from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
17
split_batch_by_proposal_len)
18
from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
@@ -48,8 +49,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
49
"""
50
51
@classmethod
51
- def from_workers(cls, proposer_worker: MultiStepWorker,
52
- scorer_worker: WorkerBase) -> "SpecDecodeWorker":
52
+ def create_worker(
53
+ cls,
54
+ scorer_worker: WorkerBase,
55
+ draft_worker_kwargs,
56
+ ) -> "SpecDecodeWorker":
57
+
58
+ if "ngram_prompt_lookup_max" in draft_worker_kwargs:
59
+ ngram_prompt_lookup_max = (
60
+ draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
61
+ ngram_prompt_lookup_min = (
62
+ draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
63
+ else:
64
+ ngram_prompt_lookup_max = 0
65
+
66
+ if ngram_prompt_lookup_max > 0:
67
+ proposer_worker = NGramWorker(**draft_worker_kwargs)
68
+ proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
69
+ ngram_prompt_lookup_max)
70
+ else:
71
+ proposer_worker = MultiStepWorker(**draft_worker_kwargs)
72
+
73
return SpecDecodeWorker(
74
proposer_worker,
75
scorer_worker,
@@ -59,7 +79,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
79
80
def __init__(
81
self,
62
- proposer_worker: MultiStepWorker,
82
+ proposer_worker: WorkerBase,
83
scorer_worker: WorkerBase,
84
rejection_sampler: RejectionSampler,
85
metrics_collector: Optional[AsyncMetricsCollector] = None,
@@ -134,8 +154,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
154
"""
155
(self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor
156
) = True
137
- (self.proposer_worker.model_runner.model.sampler.
138
- include_gpu_probs_tensor) = True
157
+ self.proposer_worker.set_include_gpu_probs_tensor()
158
159
def determine_num_available_blocks(self) -> Tuple[int, int]:
160
"""Determine the number of cache blocks to use.
@@ -183,8 +202,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
202
"speculative decoding "
203
"requires non-None seq_group_metadata_list")
204
186
- logger.info("spec_decode_worker.execute_model num_lookahead_slots=%d",
187
- num_lookahead_slots)
205
+ #logger.info("spec_decode_worker.execute_model num_lookahead_slots=%d",
206
+ # num_lookahead_slots)
207
208
# If no spec tokens, call the proposer and scorer workers normally.
209
# Used for prefill.
@@ -216,7 +235,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
235
proposer and scorer model so that the KV cache is consistent between the
236
two.
237
"""
219
- logger.info("run proposer worker no spec")
238
+ #logger.info("run proposer worker no spec")
239
240
self.proposer_worker.execute_model(
241
seq_group_metadata_list=seq_group_metadata_list,
@@ -225,7 +244,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
244
blocks_to_copy=blocks_to_copy,
245
)
246
228
- logger.info("run target worker no spec")
247
+ #logger.info("run target worker no spec")
248
sampler_output = self.scorer_worker.execute_model(
249
seq_group_metadata_list=seq_group_metadata_list,
250
blocks_to_swap_in=blocks_to_swap_in,
@@ -259,7 +278,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
278
sequence.
279
"""
280
262
- logger.info("get spec proposals")
281
+ #logger.info("get spec proposals")
282
# Generate proposals using draft worker.
283
assert blocks_to_swap_in is not None
284
assert blocks_to_swap_out is not None
@@ -268,7 +287,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
287
seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
288
blocks_to_copy, k)
289
271
- logger.info("score proposals")
290
+ #logger.info("score proposals")
291
proposal_scores = self.scorer.score_proposals(
292
seq_group_metadata_list,
293
blocks_to_swap_in,
@@ -278,11 +297,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
297
proposals,
298
)
299
281
- logger.info("verify proposals")
300
+ #logger.info("verify proposals")
301
accepted_token_ids = self._verify_tokens(seq_group_metadata_list,
302
proposal_scores, proposals, k)
303
285
- logger.info("create output list")
304
+ #logger.info("create output list")
305
return self._create_output_sampler_list(seq_group_metadata_list,
306
accepted_token_ids, k)
307
vllm/spec_decode/top1_proposer.py
new
+200
@@ -0,0 +1,200 @@
1
+from typing import Dict, List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6
+from vllm.spec_decode.interfaces import (SpeculativeProposals,
7
+ SpeculativeProposer)
8
+from vllm.spec_decode.util import sampler_output_to_torch
9
+from vllm.worker.worker_base import WorkerBase
10
+
11
+
12
+class Top1Proposer(SpeculativeProposer):
13
+ """Helper class which separates out sequences which would exceed the max
14
+ model length when speculated upon.
15
+
16
+ This allows combinations of models such as JackFram/llama-68m draft with
17
+ meta-llama/Llama2-13b-chat-hf, as llama-68m has max_position_embeddings of
18
+ 2048 while Llama2-13b has max_position_embeddings of 4096.
19
+
20
+ We treat the sequences which exceed the proposal draft model length as
21
+ "non-spec sequences". Essentially they skip the draft model and go through
22
+ normal decoding in the target model.
23
+
24
+ Currently, only proposal_lens of 0 and k are supported, where k is a global
25
+ batch proposal length. In the future vLLM should support per-sequence
26
+ proposal lengths.
27
+ """
28
+
29
+ def __init__(
30
+ self,
31
+ worker: WorkerBase,
32
+ device: str,
33
+ vocab_size: int,
34
+ max_proposal_len: Optional[int] = None,
35
+ ):
36
+ self._worker = worker
37
+ self._device = device
38
+ self.max_proposal_len = max_proposal_len
39
+ self._vocab_size = vocab_size
40
+
41
+ def get_proposals(
42
+ self,
43
+ seq_group_metadata_list: List[SequenceGroupMetadata],
44
+ blocks_to_swap_in: Dict[int, int],
45
+ blocks_to_swap_out: Dict[int, int],
46
+ blocks_to_copy: Dict[int, List[int]],
47
+ proposal_len: int,
48
+ ) -> SpeculativeProposals:
49
+ """Get speculative proposals given the input batch.
50
+
51
+ Sequences which would exceed the max model length are skipped during
52
+ speculation.
53
+ """
54
+
55
+ # Split speculative- and non-speculative- sequences.
56
+ (
57
+ proposal_lens,
58
+ nonzero_proposal_len_seqs,
59
+ nonzero_proposal_len_indices,
60
+ ) = self._split_by_max_model_len(seq_group_metadata_list, proposal_len)
61
+
62
+ if nonzero_proposal_len_seqs:
63
+ # Speculate tokens using the draft worker for the speculative
64
+ # sequences.
65
+ # If sampler_transposed is true, then maybe_sampler_output's
66
+ # token_ids is like [batch] format in proposal_len size list,
67
+ # while if it is false, the format would be [proposal_len]
68
+ # in batch size list
69
+ maybe_sampler_output, transposed = self._worker.sampler_output(
70
+ seq_group_metadata_list=nonzero_proposal_len_seqs,
71
+ blocks_to_swap_in=blocks_to_swap_in,
72
+ blocks_to_swap_out=blocks_to_swap_out,
73
+ blocks_to_copy=blocks_to_copy,
74
+ sample_len=proposal_len,
75
+ )
76
+ else:
77
+ # If no sequences can be speculated, set sampler output to None.
78
+ maybe_sampler_output = None
79
+ transposed = False
80
+
81
+ # Combine speculative- and non-speculative sequences into the same
82
+ # representation.
83
+ proposal_tokens, proposal_probs, proposal_lens = self._merge_outputs(
84
+ batch_size=len(seq_group_metadata_list),
85
+ proposal_len=proposal_len,
86
+ maybe_sampler_output=maybe_sampler_output,
87
+ proposal_lens=proposal_lens,
88
+ nonzero_proposal_len_indices=nonzero_proposal_len_indices,
89
+ sampler_transposed=transposed,
90
+ )
91
+
92
+ proposals = SpeculativeProposals(
93
+ proposal_token_ids=proposal_tokens,
94
+ proposal_probs=proposal_probs,
95
+ proposal_lens=proposal_lens,
96
+ )
97
+
98
+ return proposals
99
+
100
+ def _split_by_max_model_len(
101
+ self,
102
+ seq_group_metadata_list: List[SequenceGroupMetadata],
103
+ proposal_len: int,
104
+ ) -> Tuple[List[int], List[SequenceGroupMetadata], List[int]]:
105
+ """Determine which sequences would exceed the max model length."""
106
+
107
+ proposal_lens: List[int] = []
108
+ nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
109
+ nonzero_proposal_len_indices: List[int] = []
110
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
111
+ seq_data = next(iter(seq_group_metadata.seq_data.values()))
112
+ seq_len = seq_data.get_len()
113
+
114
+ # Currently only proposal lens of 0 or the global batch proposal len
115
+ # are supported.
116
+ # If max_proposal_len is defined, then we shall no exccess this
117
+ # quota for nonzero_proposal
118
+ if (self.max_proposal_len is None
119
+ or seq_len + proposal_len < self.max_proposal_len):
120
+ proposal_lens.append(proposal_len)
121
+ nonzero_proposal_len_seqs.append(seq_group_metadata)
122
+ nonzero_proposal_len_indices.append(i)
123
+ else:
124
+ proposal_lens.append(0)
125
+
126
+ return (
127
+ proposal_lens,
128
+ nonzero_proposal_len_seqs,
129
+ nonzero_proposal_len_indices,
130
+ )
131
+
132
+ def _merge_outputs(
133
+ self,
134
+ batch_size: int,
135
+ proposal_len: int,
136
+ maybe_sampler_output: Optional[SamplerOutput],
137
+ proposal_lens: List[int],
138
+ nonzero_proposal_len_indices: List[int],
139
+ sampler_transposed: bool,
140
+ ) -> Tuple[torch.Tensor, torch.tensor, torch.Tensor]:
141
+ """After speculations are produced, merge the speculation results with
142
+ the skipped sequences.
143
+ """
144
+ if maybe_sampler_output is None:
145
+ # If no speculative tokens, the sampler output will be None.
146
+ # In this case we return empty proposals.
147
+ proposal_tokens = torch.full(
148
+ size=(
149
+ batch_size,
150
+ proposal_len,
151
+ ),
152
+ fill_value=-1,
153
+ dtype=torch.long,
154
+ device=self._device,
155
+ )
156
+ proposal_probs = torch.zeros(
157
+ batch_size,
158
+ proposal_len,
159
+ self._vocab_size,
160
+ dtype=torch.float32,
161
+ device=self._device,
162
+ )
163
+ proposal_lens_tensor = torch.zeros(len(proposal_lens),
164
+ dtype=torch.long,
165
+ device=self._device)
166
+ return proposal_tokens, proposal_probs, proposal_lens_tensor
167
+
168
+ sampler_output = maybe_sampler_output
169
+ proposal_tokens, proposal_probs = sampler_output_to_torch(
170
+ sampler_output, sampler_transposed)
171
+
172
+ # Now, reformat the output GPU tensors such that each sequence has
173
+ # a proposal. the proposal can be empty, e.g. [-1, -1, -1]
174
+
175
+ entire_proposal_tokens = torch.full(
176
+ size=(batch_size, *proposal_tokens.shape[1:]),
177
+ fill_value=-1,
178
+ dtype=torch.long,
179
+ device=self._device,
180
+ )
181
+ entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens
182
+ entire_proposal_probs = torch.zeros(
183
+ batch_size,
184
+ *proposal_probs.shape[1:],
185
+ dtype=torch.float32,
186
+ device=self._device,
187
+ )
188
+ entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
189
+
190
+ proposal_tokens, proposal_probs = (
191
+ entire_proposal_tokens,
192
+ entire_proposal_probs,
193
+ )
194
+
195
+ proposal_lens_tensor = torch.zeros(batch_size,
196
+ dtype=torch.long,
197
+ device=self._device)
198
+ proposal_lens_tensor[nonzero_proposal_len_indices] = proposal_len
199
+
200
+ return proposal_tokens, proposal_probs, proposal_lens_tensor
vllm/spec_decode/util.py
+12
-4
@@ -49,10 +49,13 @@ def split_batch_by_proposal_len(
49
50
51
def sampler_output_to_torch(
52
- sampler_output_list: List[SamplerOutput],
53
-) -> Tuple[torch.Tensor, torch.Tensor]:
52
+ sampler_output_list: List[SamplerOutput],
53
+ sampler_transposed: bool) -> Tuple[torch.Tensor, torch.Tensor]:
54
"""Utility function which converts a list of SamplerOutput to tensors.
55
56
+ sampler_transposed here is used as the indicator for whether
57
+ we need do additional tensor transpose logic here.
58
+
59
Returns:
60
sampled_token_ids: torch.Tensor
61
shape: [batch_size, len(sampler_output_list)]
@@ -68,7 +71,10 @@ def sampler_output_to_torch(
71
for sampler_output in sampler_output_list
72
],
73
dim=0,
71
- ).transpose(0, 1)
74
+ )
75
+
76
+ if sampler_transposed:
77
+ sampled_token_probs = sampled_token_probs.transpose(0, 1)
78
79
# shape: [batch_size, num_sampler_output]
80
sampled_token_ids = torch.stack(
@@ -77,7 +83,9 @@ def sampler_output_to_torch(
83
for sampler_output in sampler_output_list
84
],
85
dim=0,
80
- ).transpose(0, 1)
86
+ )
87
+ if sampler_transposed:
88
+ sampled_token_ids = sampled_token_ids.transpose(0, 1)
89
90
return sampled_token_ids, sampled_token_probs
91