A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Speculative decoding 3/9] Worker which speculates, scores, and applies rejection sampling (#3103)
Cade Daniel committed
Mar 8, 2024 at 23:32 UTC
8437bae6ef47a690d18c72f0da02c7e5abe83866
21 files changed
+2786
-215
.buildkite/test-pipeline.yaml
+4
-1
@@ -28,7 +28,7 @@ steps:
28
num_gpus: 2 # only support 1 or 2 for now.
29
30
- label: Engine Test
31
- command: pytest -v -s engine
31
+ command: pytest -v -s engine test_sequence.py
32
33
- label: Entrypoints Test
34
command: pytest -v -s entrypoints
@@ -52,6 +52,9 @@ steps:
52
- label: Worker Test
53
command: pytest -v -s worker
54
55
+- label: Speculative decoding tests
56
+ command: pytest -v -s spec_decode
57
+
58
- label: LoRA Test
59
command: pytest -v -s lora --forked
60
tests/spec_decode/__init__.py
renamed
tests/spec_decode/test_batch_expansion.py
new
+95
@@ -0,0 +1,95 @@
1
+import torch
2
+import pytest
3
+
4
+from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
5
+
6
+from .utils import mock_worker, create_seq_group_metadata_from_prompts
7
+
8
+
9
+@pytest.mark.parametrize('num_target_seq_ids', [100])
10
+def test_create_target_seq_id_iterator(num_target_seq_ids: int):
11
+ """Verify all new sequence ids are greater than all input
12
+ seq ids.
13
+ """
14
+ scorer = BatchExpansionTop1Scorer(mock_worker(), 'cuda:0', 32_000)
15
+
16
+ all_seq_ids = [
17
+ [1, 3, 5, 7],
18
+ list(range(100)) + [0],
19
+ [100],
20
+ ]
21
+
22
+ for seq_ids in all_seq_ids:
23
+ max_seq_id = max(seq_ids)
24
+ iterator = scorer._create_target_seq_id_iterator(seq_ids) # pylint: disable=protected-access
25
+ for _ in range(num_target_seq_ids):
26
+ assert next(iterator) > max_seq_id
27
+
28
+
29
+@pytest.mark.parametrize('k', [1, 2, 6])
30
+def test_get_token_ids_to_score(k: int):
31
+ """Verify correct tokens are selected for scoring.
32
+ """
33
+ proposal_token_ids = torch.tensor(
34
+ list(range(k)),
35
+ dtype=torch.int64,
36
+ device='cuda',
37
+ )
38
+
39
+ expected_output = [
40
+ [],
41
+ ]
42
+ for i in range(proposal_token_ids.shape[0]):
43
+ expected_output.append(proposal_token_ids[:i + 1].tolist())
44
+
45
+ scorer = BatchExpansionTop1Scorer(mock_worker(), 'cuda:0', 32_000)
46
+ actual_output = scorer._get_token_ids_to_score(proposal_token_ids) # pylint: disable=protected-access
47
+
48
+ actual_output = [
49
+ x.tolist() if isinstance(x, torch.Tensor) else x for x in actual_output
50
+ ]
51
+
52
+ assert actual_output == expected_output
53
+
54
+
55
+@pytest.mark.parametrize('k', [1, 2, 6])
56
+def test_create_single_target_seq_group_metadata(k: int):
57
+ """Verify correct creation of a batch-expanded seq group metadata.
58
+ """
59
+
60
+ prompt_tokens = [1, 2, 3]
61
+ prev_output_tokens = [4, 5, 6]
62
+
63
+ token_ids = list(range(k))
64
+
65
+ num_tokens_processed = len(prompt_tokens) + len(prev_output_tokens) - 1
66
+
67
+ final_seq_len = len(prompt_tokens) + len(prev_output_tokens) + len(
68
+ token_ids)
69
+
70
+ block_size = 32
71
+ input_seq_group_metadata = create_seq_group_metadata_from_prompts(
72
+ [prompt_tokens], 2048 // block_size, block_size, [final_seq_len],
73
+ [prev_output_tokens], [num_tokens_processed])[0]
74
+
75
+ input_seq_id = list(input_seq_group_metadata.seq_data.keys())[0]
76
+ target_seq_id = 100
77
+
78
+ scorer = BatchExpansionTop1Scorer(mock_worker(), 'cuda:0', 32_000)
79
+ output = scorer._create_single_target_seq_group_metadata( # pylint: disable=protected-access
80
+ input_seq_group_metadata,
81
+ input_seq_id,
82
+ target_seq_id,
83
+ token_ids,
84
+ )
85
+
86
+ assert output.request_id == input_seq_group_metadata.request_id
87
+ assert len(output.seq_data) == 1
88
+ assert output.seq_data[target_seq_id].get_prompt_token_ids(
89
+ ) == prompt_tokens
90
+ assert output.seq_data[target_seq_id].get_output_token_ids(
91
+ ) == prev_output_tokens + token_ids
92
+
93
+ assert len(output.block_tables) == 1
94
+ assert output.block_tables[
95
+ target_seq_id] == input_seq_group_metadata.block_tables[input_seq_id]
tests/spec_decode/test_metrics.py
new
+157
@@ -0,0 +1,157 @@
1
+import torch
2
+import math
3
+import pytest
4
+
5
+from unittest.mock import MagicMock
6
+
7
+from vllm.spec_decode.metrics import AsyncMetricsCollector
8
+
9
+
10
+def test_initial_call_returns_none():
11
+ """Expect first call to get metrics to return None.
12
+ """
13
+ rej_sampler = MagicMock()
14
+ rej_sampler.num_accepted_tokens = torch.tensor(0,
15
+ dtype=torch.long,
16
+ device='cuda')
17
+ rej_sampler.num_emitted_tokens = torch.tensor(0,
18
+ dtype=torch.long,
19
+ device='cuda')
20
+ rej_sampler.num_draft_tokens = 0
21
+
22
+ collector = AsyncMetricsCollector(rej_sampler)
23
+ collector.init_gpu_tensors(rank=0)
24
+ maybe_metrics = collector.maybe_collect_rejsample_metrics(k=5)
25
+ assert maybe_metrics is None
26
+
27
+
28
+def test_second_call_returns_metrics():
29
+ """Expect second call to not return None.
30
+ """
31
+ rej_sampler = MagicMock()
32
+ rej_sampler.num_accepted_tokens = torch.tensor(0,
33
+ dtype=torch.long,
34
+ device='cuda')
35
+ rej_sampler.num_emitted_tokens = torch.tensor(0,
36
+ dtype=torch.long,
37
+ device='cuda')
38
+ rej_sampler.num_draft_tokens = 0
39
+
40
+ collect_interval_s = 5.0
41
+ timer = MagicMock()
42
+ timer.side_effect = [
43
+ 0.0, collect_interval_s + 0.1, collect_interval_s + 0.2
44
+ ]
45
+
46
+ collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
47
+ timer=timer,
48
+ collect_interval_s=collect_interval_s)
49
+ collector.init_gpu_tensors(rank=0)
50
+ _ = collector.maybe_collect_rejsample_metrics(k=5)
51
+ metrics = collector.maybe_collect_rejsample_metrics(k=5)
52
+ assert metrics is not None
53
+
54
+
55
+@pytest.mark.parametrize("rank", [1, 2, 3, 4])
56
+def test_nonzero_rank_noop(rank):
57
+ """Verify nonzero ranks don't collect metrics.
58
+ """
59
+ rej_sampler = MagicMock()
60
+ rej_sampler.num_accepted_tokens = torch.tensor(0,
61
+ dtype=torch.long,
62
+ device='cuda')
63
+ rej_sampler.num_emitted_tokens = torch.tensor(0,
64
+ dtype=torch.long,
65
+ device='cuda')
66
+ rej_sampler.num_draft_tokens = 0
67
+
68
+ collector = AsyncMetricsCollector(rej_sampler)
69
+ collector.init_gpu_tensors(rank=rank)
70
+ _ = collector.maybe_collect_rejsample_metrics(k=5)
71
+ metrics = collector.maybe_collect_rejsample_metrics(k=5)
72
+ assert metrics is None
73
+
74
+
75
+def test_noop_until_time():
76
+ """Verify metrics aren't collected until enough time passes.
77
+ """
78
+ rej_sampler = MagicMock()
79
+ rej_sampler.num_accepted_tokens = torch.tensor(0,
80
+ dtype=torch.long,
81
+ device='cuda')
82
+ rej_sampler.num_emitted_tokens = torch.tensor(0,
83
+ dtype=torch.long,
84
+ device='cuda')
85
+ rej_sampler.num_draft_tokens = 0
86
+
87
+ collect_interval_s = 5.0
88
+ timer = MagicMock()
89
+ timer.side_effect = [
90
+ 0.0, collect_interval_s - 0.1, collect_interval_s - 0.1,
91
+ collect_interval_s + 0.1, collect_interval_s + 0.1
92
+ ]
93
+
94
+ collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
95
+ timer=timer,
96
+ collect_interval_s=collect_interval_s)
97
+ collector.init_gpu_tensors(rank=0)
98
+
99
+ _ = collector.maybe_collect_rejsample_metrics(k=5)
100
+ metrics = collector.maybe_collect_rejsample_metrics(k=5)
101
+ assert metrics is None
102
+
103
+ _ = collector.maybe_collect_rejsample_metrics(k=5)
104
+ metrics = collector.maybe_collect_rejsample_metrics(k=5)
105
+ assert metrics is not None
106
+
107
+
108
+@pytest.mark.parametrize("has_data", [True, False])
109
+def test_initial_metrics_has_correct_values(has_data: bool):
110
+ """Test correctness of metrics data.
111
+ """
112
+ if has_data:
113
+ num_accepted_tokens = 103
114
+ num_emitted_tokens = 104
115
+ num_draft_tokens = 105
116
+ else:
117
+ num_accepted_tokens = 0
118
+ num_emitted_tokens = 0
119
+ num_draft_tokens = 0
120
+ k = 5
121
+
122
+ num_possible_tokens = AsyncMetricsCollector.get_max_num_accepted_tokens(
123
+ num_draft_tokens, k)
124
+
125
+ rej_sampler = MagicMock()
126
+ rej_sampler.num_accepted_tokens = torch.tensor(num_accepted_tokens,
127
+ dtype=torch.long,
128
+ device='cuda')
129
+ rej_sampler.num_emitted_tokens = torch.tensor(num_emitted_tokens,
130
+ dtype=torch.long,
131
+ device='cuda')
132
+ rej_sampler.num_draft_tokens = num_draft_tokens
133
+
134
+ collect_interval_s = 5.0
135
+ timer = MagicMock()
136
+ timer.side_effect = [
137
+ 0.0, collect_interval_s + 0.1, collect_interval_s + 0.2
138
+ ]
139
+
140
+ collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
141
+ timer=timer,
142
+ collect_interval_s=collect_interval_s)
143
+ collector.init_gpu_tensors(rank=0)
144
+ _ = collector.maybe_collect_rejsample_metrics(k)
145
+ metrics = collector.maybe_collect_rejsample_metrics(k)
146
+
147
+ assert metrics.num_spec_tokens == k
148
+ assert metrics.accepted_tokens == num_accepted_tokens
149
+ assert metrics.draft_tokens == num_draft_tokens
150
+ assert metrics.emitted_tokens == num_emitted_tokens
151
+
152
+ if has_data:
153
+ assert metrics.draft_acceptance_rate == num_accepted_tokens / num_draft_tokens
154
+ assert metrics.system_efficiency == num_emitted_tokens / num_possible_tokens
155
+ else:
156
+ assert math.isnan(metrics.draft_acceptance_rate)
157
+ assert math.isnan(metrics.system_efficiency)
tests/spec_decode/test_multi_step_worker.py
renamed
+160
-2
@@ -3,14 +3,15 @@ import random
3
import pytest
4
from unittest.mock import MagicMock
5
6
-from vllm.worker.spec_decode.multi_step_worker import MultiStepWorker
6
+from vllm.spec_decode.multi_step_worker import MultiStepWorker, DraftModelTop1Proposer
7
from vllm.worker.worker import Worker
8
from vllm.model_executor.utils import set_random_seed
9
+from vllm.sequence import SamplerOutput
10
11
from .utils import (create_execute_model_data, create_worker,
12
create_seq_group_metadata_from_prompts, zero_kv_cache,
13
patch_execute_model_with_seeds,
13
- assert_logprobs_dict_allclose)
14
+ assert_logprobs_dict_allclose, create_batch)
15
16
17
@pytest.mark.parametrize('num_steps', list(range(1, 17)))
@@ -259,3 +260,160 @@ def test_same_output_for_multi_step():
260
multi_step_output_logprobs, single_step_output_logprobs):
261
assert_logprobs_dict_allclose(multi_step_logprobs,
262
single_step_logprobs)
263
+
264
+
265
+@torch.inference_mode()
266
+def test_draft_proposals_full_speculation_len():
267
+ """Verify DraftModelTop1Proposer correctly handles case where all sequences
268
+ can speculate.
269
+ """
270
+ k = 10
271
+ batch_size = 32
272
+ vocab_size = 32_000
273
+ device = 'cuda:0'
274
+
275
+ draft_worker = MagicMock()
276
+ proposer = DraftModelTop1Proposer(
277
+ draft_worker=draft_worker,
278
+ device=device,
279
+ max_model_len=2048,
280
+ vocab_size=vocab_size,
281
+ )
282
+ draft_worker.execute_model_multi_step.return_value = [
283
+ SamplerOutput(
284
+ outputs=[],
285
+ sampled_token_probs=torch.rand(batch_size,
286
+ vocab_size,
287
+ device=device,
288
+ dtype=torch.float32),
289
+ sampled_token_ids=torch.randint(low=0,
290
+ high=vocab_size,
291
+ size=(batch_size, ),
292
+ device=device,
293
+ dtype=torch.long),
294
+ ) for _ in range(k)
295
+ ]
296
+
297
+ execute_model_data, _, _ = create_batch(batch_size, k)
298
+
299
+ proposals = proposer.get_proposals(
300
+ **execute_model_data.to_dict(),
301
+ max_proposal_len=k,
302
+ )
303
+
304
+ assert torch.is_tensor(proposals.proposal_token_ids)
305
+ assert torch.is_tensor(proposals.proposal_probs)
306
+
307
+ assert proposals.proposal_token_ids.shape == torch.Size([batch_size, k])
308
+ assert proposals.proposal_probs.shape[:-1] == torch.Size([batch_size, k])
309
+
310
+ assert proposals.proposal_lens.shape == torch.Size([batch_size])
311
+ assert proposals.proposal_lens.tolist() == [k for _ in range(batch_size)]
312
+
313
+
314
+@torch.inference_mode()
315
+def test_draft_proposals_no_speculations():
316
+ """Verify DraftModelTop1Proposer correctly handles case where no sequences
317
+ can speculate.
318
+ """
319
+ k = 10
320
+ batch_size = 32
321
+ vocab_size = 32_000
322
+ device = 'cuda:0'
323
+ prompt_len = 10
324
+
325
+ draft_worker = MagicMock()
326
+ proposer = DraftModelTop1Proposer(
327
+ draft_worker=draft_worker,
328
+ device=device,
329
+ max_model_len=prompt_len + k - 1,
330
+ vocab_size=vocab_size,
331
+ )
332
+
333
+ execute_model_data, _, _ = create_batch(batch_size,
334
+ k,
335
+ prompt_len=prompt_len)
336
+
337
+ proposals = proposer.get_proposals(
338
+ **execute_model_data.to_dict(),
339
+ max_proposal_len=k,
340
+ )
341
+
342
+ assert torch.is_tensor(proposals.proposal_token_ids)
343
+ assert torch.is_tensor(proposals.proposal_probs)
344
+
345
+ assert proposals.proposal_token_ids.shape == torch.Size([0, k])
346
+ assert proposals.proposal_probs.shape[:-1] == torch.Size([0, k])
347
+
348
+ assert proposals.proposal_lens.shape == torch.Size([batch_size])
349
+ assert proposals.proposal_lens.tolist() == [0 for _ in range(batch_size)]
350
+
351
+
352
+@torch.inference_mode()
353
+def test_draft_proposals_mixed_k():
354
+ """Verify DraftModelTop1Proposer correctly handles case some sequences can
355
+ speculate and some can't.
356
+ """
357
+ k = 10
358
+ batch_size = 32
359
+ vocab_size = 32_000
360
+ device = 'cuda:0'
361
+
362
+ small_prompt_len = 5
363
+ long_prompt_len = 10
364
+ prev_output_token_len = 20
365
+
366
+ expected_num_proposal_seqs = 6
367
+ expected_num_no_proposal_seqs = batch_size - expected_num_proposal_seqs
368
+
369
+ prompt_len = [
370
+ small_prompt_len for _ in range(expected_num_proposal_seqs - 1)
371
+ ] + [long_prompt_len
372
+ for _ in range(expected_num_no_proposal_seqs)] + [small_prompt_len]
373
+
374
+ draft_worker = MagicMock()
375
+ proposer = DraftModelTop1Proposer(
376
+ draft_worker=draft_worker,
377
+ device=device,
378
+ max_model_len=long_prompt_len + prev_output_token_len + k - 1,
379
+ vocab_size=vocab_size,
380
+ )
381
+
382
+ draft_worker.execute_model_multi_step.return_value = [
383
+ SamplerOutput(
384
+ outputs=[],
385
+ sampled_token_probs=torch.rand(expected_num_proposal_seqs,
386
+ vocab_size,
387
+ device=device,
388
+ dtype=torch.float32),
389
+ sampled_token_ids=torch.randint(
390
+ low=0,
391
+ high=vocab_size,
392
+ size=(expected_num_proposal_seqs, ),
393
+ device=device,
394
+ dtype=torch.long),
395
+ ) for _ in range(k)
396
+ ]
397
+
398
+ execute_model_data, _, _ = create_batch(
399
+ batch_size,
400
+ k,
401
+ prompt_len=prompt_len,
402
+ prev_output_token_len=prev_output_token_len,
403
+ )
404
+
405
+ proposals = proposer.get_proposals(
406
+ **execute_model_data.to_dict(),
407
+ max_proposal_len=k,
408
+ )
409
+
410
+ assert torch.is_tensor(proposals.proposal_token_ids)
411
+ assert torch.is_tensor(proposals.proposal_probs)
412
+
413
+ assert proposals.proposal_token_ids.shape == torch.Size([batch_size, k])
414
+ assert proposals.proposal_probs.shape[:-1] == torch.Size([batch_size, k])
415
+
416
+ assert proposals.proposal_lens.shape == torch.Size([batch_size])
417
+ assert proposals.proposal_lens.tolist() == [
418
+ k for _ in range(expected_num_proposal_seqs - 1)
419
+ ] + [0 for _ in range(expected_num_no_proposal_seqs)] + [k]
tests/spec_decode/test_spec_decode_worker.py
new
+591
@@ -0,0 +1,591 @@
1
+import torch
2
+import random
3
+import pytest
4
+from unittest.mock import MagicMock
5
+
6
+from vllm.spec_decode.multi_step_worker import MultiStepWorker
7
+from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker, split_num_cache_blocks_evenly
8
+from vllm.spec_decode.interfaces import SpeculativeProposals
9
+from vllm.model_executor.utils import set_random_seed
10
+from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11
+from .utils import mock_worker, create_batch, ExecuteModelData, create_sampler_output_list
12
+from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics, AsyncMetricsCollector
13
+
14
+
15
+@pytest.mark.parametrize('k', [1, 2, 6])
16
+@pytest.mark.parametrize('batch_size', [1, 2, 32])
17
+@torch.inference_mode()
18
+def test_correctly_calls_draft_model(k: int, batch_size: int):
19
+ """Verify SpecDecodeWorker calls the draft worker with correct
20
+ inputs. Everything else is mocked out.
21
+ """
22
+ draft_worker = mock_worker(cls=MultiStepWorker)
23
+ target_worker = mock_worker()
24
+ rejection_sampler = MagicMock(spec=RejectionSampler)
25
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
26
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
27
+ metrics_collector)
28
+
29
+ exception_secret = 'artifical stop'
30
+ draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
31
+
32
+ execute_model_data, _, _ = create_batch(batch_size, k)
33
+
34
+ with pytest.raises(ValueError, match=exception_secret):
35
+ worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
36
+
37
+ call_args_list = draft_worker.get_spec_proposals.call_args_list
38
+ assert len(call_args_list) == 1
39
+
40
+ for args, _ in call_args_list:
41
+ (seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
42
+ blocks_to_copy, actual_k) = args
43
+ actual_execute_model_data = ExecuteModelData(seq_group_metadata_list,
44
+ blocks_to_swap_in,
45
+ blocks_to_swap_out,
46
+ blocks_to_copy)
47
+ assert actual_execute_model_data == execute_model_data
48
+ assert actual_k == k
49
+
50
+
51
+@pytest.mark.parametrize('k', [1, 2, 6])
52
+@pytest.mark.parametrize('batch_size', [1, 2, 32])
53
+@torch.inference_mode()
54
+def test_correctly_calls_target_model(k: int, batch_size: int):
55
+ """Verify SpecDecodeWorker calls the target model with correct
56
+ inputs. Everything else is mocked out.
57
+ """
58
+ draft_worker = mock_worker(cls=MultiStepWorker)
59
+ target_worker = mock_worker()
60
+ rejection_sampler = MagicMock(spec=RejectionSampler)
61
+ rejection_sampler.token_id_dtype = torch.int64
62
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
63
+
64
+ draft_worker.device = 'cuda'
65
+ target_worker.device = 'cuda'
66
+
67
+ set_random_seed(1)
68
+
69
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
70
+ metrics_collector)
71
+ worker.init_model()
72
+
73
+ vocab_size = 32_000
74
+
75
+ proposal_token_ids = torch.randint(low=0,
76
+ high=vocab_size,
77
+ size=(batch_size, k),
78
+ dtype=torch.int64,
79
+ device='cuda')
80
+ proposal_probs = torch.rand(batch_size,
81
+ k,
82
+ vocab_size,
83
+ dtype=torch.float32,
84
+ device='cuda')
85
+ proposal_lens = torch.ones(batch_size, dtype=torch.int64,
86
+ device='cuda') * k
87
+
88
+ execute_model_data, prompts, prev_output_tokens = create_batch(
89
+ batch_size, k)
90
+
91
+ draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
92
+ proposal_token_ids=proposal_token_ids,
93
+ proposal_probs=proposal_probs,
94
+ proposal_lens=proposal_lens)
95
+
96
+ exception_secret = 'artifical stop'
97
+ target_worker.execute_model.side_effect = ValueError(exception_secret)
98
+
99
+ with pytest.raises(ValueError, match=exception_secret):
100
+ worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
101
+
102
+ seen_contexts = []
103
+
104
+ call_args_list = target_worker.execute_model.call_args_list
105
+ assert len(call_args_list) == 1
106
+ for args, kwargs in call_args_list:
107
+ target_execute_model_data = ExecuteModelData.from_dict(kwargs)
108
+
109
+ assert len(target_execute_model_data.seq_group_metadata_list) == (
110
+ k + 1) * batch_size
111
+ for seq_group_metadata in (
112
+ target_execute_model_data.seq_group_metadata_list):
113
+ for seq_data in seq_group_metadata.seq_data.values():
114
+ seen_contexts.append(seq_data.get_token_ids())
115
+
116
+ expected_seen_contexts = []
117
+
118
+ for prompt, prev_generated, draft_tokens in zip(
119
+ prompts, prev_output_tokens, proposal_token_ids.tolist()):
120
+
121
+ for i in range(len(draft_tokens) + 1):
122
+ expected_seen_contexts.append(prompt + prev_generated +
123
+ draft_tokens[:i])
124
+
125
+ seen_contexts.sort()
126
+ expected_seen_contexts.sort()
127
+ assert expected_seen_contexts == seen_contexts
128
+
129
+
130
+@pytest.mark.parametrize('k', [1, 2, 6])
131
+@pytest.mark.parametrize('batch_size', [1, 2, 32])
132
+@torch.inference_mode()
133
+def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
134
+ """Verify SpecDecodeWorker calls the rejection sampler with
135
+ correct inputs. Everything else is mocked out.
136
+ """
137
+ vocab_size = 32_000
138
+
139
+ draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
140
+ target_worker = mock_worker(vocab_size=vocab_size)
141
+ rejection_sampler = MagicMock(spec=RejectionSampler)
142
+ rejection_sampler.token_id_dtype = torch.int64
143
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
144
+ draft_worker.device = 'cuda'
145
+ target_worker.device = 'cuda'
146
+
147
+ set_random_seed(1)
148
+
149
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
150
+ metrics_collector)
151
+ worker.init_model()
152
+
153
+ proposal_token_ids = torch.randint(low=0,
154
+ high=vocab_size,
155
+ size=(batch_size, k),
156
+ dtype=torch.int64,
157
+ device='cuda')
158
+ proposal_probs = torch.rand(batch_size,
159
+ k,
160
+ vocab_size,
161
+ dtype=torch.float32,
162
+ device='cuda')
163
+
164
+ proposal_lens = torch.ones(batch_size, dtype=torch.int64,
165
+ device='cuda') * k
166
+
167
+ execute_model_data, _, _ = create_batch(batch_size, k)
168
+
169
+ draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
170
+ proposal_token_ids=proposal_token_ids,
171
+ proposal_probs=proposal_probs,
172
+ proposal_lens=proposal_lens)
173
+
174
+ target_token_ids = torch.randint(low=0,
175
+ high=vocab_size,
176
+ size=(1, batch_size * (k + 1)),
177
+ dtype=torch.int64,
178
+ device='cuda')
179
+ target_token_probs = torch.rand(1,
180
+ batch_size * (k + 1),
181
+ vocab_size,
182
+ dtype=torch.float32,
183
+ device='cuda')
184
+ target_output = create_sampler_output_list(target_token_ids,
185
+ target_token_probs)
186
+
187
+ target_worker.execute_model.return_value = target_output[0]
188
+
189
+ exception_secret = 'artifical stop'
190
+ rejection_sampler.side_effect = ValueError(exception_secret)
191
+
192
+ with pytest.raises(ValueError, match=exception_secret):
193
+ worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
194
+
195
+ assert len(rejection_sampler.call_args_list) == 1
196
+ args, _ = rejection_sampler.call_args_list[0]
197
+ (actual_proposal_scores, actual_bonus_token_ids, actual_proposal_probs,
198
+ actual_proposal_token_ids) = args
199
+
200
+ assert torch.equal(actual_bonus_token_ids,
201
+ target_token_ids.reshape(batch_size, k + 1)[:, -1:])
202
+ assert torch.equal(
203
+ actual_proposal_scores,
204
+ target_token_probs.reshape(batch_size, k + 1, -1)[:, :-1])
205
+ assert torch.equal(actual_proposal_token_ids, proposal_token_ids)
206
+ assert torch.equal(actual_proposal_probs, proposal_probs)
207
+
208
+
209
+@pytest.mark.parametrize('k', [1, 2, 6])
210
+@pytest.mark.parametrize('batch_size', [1, 2, 32])
211
+@torch.inference_mode()
212
+def test_correctly_formats_output(k: int, batch_size: int):
213
+ """Verify SpecDecodeWorker formats sampler output correctly.
214
+ Everything else is mocked out.
215
+ """
216
+ vocab_size = 32_000
217
+
218
+ draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
219
+ target_worker = mock_worker(vocab_size=vocab_size)
220
+ rejection_sampler = MagicMock(spec=RejectionSampler)
221
+ rejection_sampler.token_id_dtype = torch.int64
222
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
223
+ draft_worker.device = 'cuda'
224
+ target_worker.device = 'cuda'
225
+
226
+ set_random_seed(1)
227
+
228
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
229
+ metrics_collector)
230
+ worker.init_model()
231
+
232
+ proposal_token_ids = torch.randint(low=0,
233
+ high=vocab_size,
234
+ size=(batch_size, k),
235
+ dtype=torch.int64,
236
+ device='cuda')
237
+ proposal_probs = torch.rand(batch_size,
238
+ k,
239
+ vocab_size,
240
+ dtype=torch.float32,
241
+ device='cuda')
242
+
243
+ proposal_lens = torch.ones(batch_size, dtype=torch.int64,
244
+ device='cuda') * k
245
+
246
+ execute_model_data, _, _ = create_batch(batch_size, k)
247
+
248
+ draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
249
+ proposal_token_ids=proposal_token_ids,
250
+ proposal_probs=proposal_probs,
251
+ proposal_lens=proposal_lens)
252
+
253
+ target_token_ids = torch.randint(low=0,
254
+ high=vocab_size,
255
+ size=(1, batch_size * (k + 1)),
256
+ dtype=torch.int64,
257
+ device='cuda')
258
+ target_token_probs = torch.rand(1,
259
+ batch_size * (k + 1),
260
+ vocab_size,
261
+ dtype=torch.float32,
262
+ device='cuda')
263
+ target_output = create_sampler_output_list(target_token_ids,
264
+ target_token_probs)
265
+
266
+ target_worker.execute_model.return_value = target_output[0]
267
+
268
+ rejection_sampler_output = torch.randint(low=0,
269
+ high=vocab_size,
270
+ size=(batch_size, k + 1),
271
+ dtype=torch.int64,
272
+ device='cuda')
273
+ for i in range(batch_size):
274
+ minimum_accepted_tokens = 1
275
+ rejection_sampler_output[i][
276
+ -random.randint(minimum_accepted_tokens, k + 1):] = -1
277
+
278
+ rejection_sampler.return_value = rejection_sampler_output
279
+
280
+ output = worker.execute_model(**execute_model_data.to_dict(),
281
+ num_spec_tokens=k)
282
+
283
+ expected_output = create_sampler_output_list(
284
+ rejection_sampler_output.transpose(0, 1), [None for _ in range(k + 1)])
285
+
286
+ seq_ids = [
287
+ next(iter(seq_group_metadata.seq_data.keys()))
288
+ for seq_group_metadata in execute_model_data.seq_group_metadata_list
289
+ ]
290
+ actual_output_by_seq = {seq_id: [] for seq_id in seq_ids}
291
+ expected_output_by_seq = {seq_id: [] for seq_id in seq_ids}
292
+
293
+ for step in output:
294
+ for seq_group in step:
295
+ for sample in seq_group.samples:
296
+ seq_id = sample.parent_seq_id
297
+ actual_output_by_seq[seq_id].append(sample)
298
+
299
+ for step in expected_output:
300
+ for seq_group in step:
301
+ for sample in seq_group.samples:
302
+ seq_id = sample.parent_seq_id
303
+ expected_output_by_seq[seq_id].append(sample)
304
+
305
+ all_seen_seq_ids = set(
306
+ list(actual_output_by_seq.keys()) +
307
+ list(expected_output_by_seq.keys()))
308
+ for seq_id in all_seen_seq_ids:
309
+ actual_by_step = actual_output_by_seq[seq_id]
310
+ expected_by_step = expected_output_by_seq[seq_id]
311
+
312
+ for i in range(k + 1):
313
+ if i >= len(actual_by_step):
314
+ assert expected_by_step[i].output_token == -1
315
+ continue
316
+ assert actual_by_step[i].output_token == expected_by_step[
317
+ i].output_token
318
+ assert actual_by_step[i].logprobs == expected_by_step[i].logprobs
319
+
320
+
321
+@pytest.mark.parametrize('k', [1, 2])
322
+@pytest.mark.parametrize('batch_size', [1])
323
+@pytest.mark.parametrize('returns_metrics', [True, False])
324
+@torch.inference_mode()
325
+def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
326
+ """Verify SpecDecodeWorker collects metrics.
327
+ """
328
+ vocab_size = 32_000
329
+
330
+ draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
331
+ target_worker = mock_worker(vocab_size=vocab_size)
332
+ rejection_sampler = MagicMock(spec=RejectionSampler)
333
+ rejection_sampler.token_id_dtype = torch.int64
334
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
335
+ draft_worker.device = 'cuda'
336
+ target_worker.device = 'cuda'
337
+
338
+ set_random_seed(1)
339
+
340
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
341
+ metrics_collector)
342
+ worker.init_model()
343
+
344
+ proposal_token_ids = torch.randint(low=0,
345
+ high=vocab_size,
346
+ size=(batch_size, k),
347
+ dtype=torch.int64,
348
+ device='cuda')
349
+ proposal_probs = torch.rand(batch_size,
350
+ k,
351
+ vocab_size,
352
+ dtype=torch.float32,
353
+ device='cuda')
354
+
355
+ proposal_lens = torch.ones(batch_size, dtype=torch.int64,
356
+ device='cuda') * k
357
+
358
+ execute_model_data, _, _ = create_batch(batch_size, k)
359
+
360
+ draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
361
+ proposal_token_ids=proposal_token_ids,
362
+ proposal_probs=proposal_probs,
363
+ proposal_lens=proposal_lens)
364
+
365
+ target_token_ids = torch.randint(low=0,
366
+ high=vocab_size,
367
+ size=(1, batch_size * (k + 1)),
368
+ dtype=torch.int64,
369
+ device='cuda')
370
+ target_token_probs = torch.rand(1,
371
+ batch_size * (k + 1),
372
+ vocab_size,
373
+ dtype=torch.float32,
374
+ device='cuda')
375
+ target_output = create_sampler_output_list(target_token_ids,
376
+ target_token_probs)
377
+
378
+ target_worker.execute_model.return_value = target_output[0]
379
+
380
+ rejection_sampler_output = torch.randint(low=0,
381
+ high=vocab_size,
382
+ size=(batch_size, k + 1),
383
+ dtype=torch.int64,
384
+ device='cuda')
385
+ for i in range(batch_size):
386
+ minimum_accepted_tokens = 1
387
+ rejection_sampler_output[i][
388
+ -random.randint(minimum_accepted_tokens, k + 1):] = -1
389
+
390
+ rejection_sampler.return_value = rejection_sampler_output
391
+
392
+ mock_rejsample_metrics = MagicMock(
393
+ spec=SpecDecodeWorkerMetrics) if returns_metrics else None
394
+ metrics_collector.maybe_collect_rejsample_metrics.return_value = mock_rejsample_metrics
395
+
396
+ output = worker.execute_model(**execute_model_data.to_dict(),
397
+ num_spec_tokens=k)
398
+ assert output[0].spec_decode_worker_metrics == mock_rejsample_metrics
399
+
400
+ call_args_list = metrics_collector.maybe_collect_rejsample_metrics.call_args_list
401
+ assert len(call_args_list) == 1
402
+ args, kwargs = call_args_list[0]
403
+ assert args[0] == k or kwargs.get('k', -1) == k
404
+
405
+
406
+@pytest.mark.parametrize('k', [0])
407
+@pytest.mark.parametrize('batch_size', [1, 2, 32])
408
+@torch.inference_mode()
409
+def test_k_equals_zero(k: int, batch_size: int):
410
+ """Verify that the SpecDecodeWorker calls the draft and target workers
411
+ when k is zero. This happens during prefill.
412
+ """
413
+ draft_worker = mock_worker(cls=MultiStepWorker)
414
+ target_worker = mock_worker()
415
+ rejection_sampler = MagicMock(spec=RejectionSampler)
416
+ rejection_sampler.token_id_dtype = torch.int64
417
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
418
+
419
+ draft_worker.device = 'cuda'
420
+ target_worker.device = 'cuda'
421
+
422
+ set_random_seed(1)
423
+
424
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
425
+ metrics_collector)
426
+
427
+ execute_model_data, prompts, prev_output_tokens = create_batch(
428
+ batch_size, k, prev_output_token_len=0)
429
+
430
+ out = worker.execute_model(**execute_model_data.to_dict(),
431
+ num_spec_tokens=k)
432
+
433
+ assert len(out) == 1, f"expected only one token output when {k=}"
434
+ assert out[0].probs is None, "expect gpu tensor references to be None"
435
+ assert out[
436
+ 0].sampled_tokens is None, "expect gpu tensor references to be None"
437
+
438
+ draft_worker.execute_model.assert_called_once_with(
439
+ **execute_model_data.to_dict(), return_python_output=False)
440
+ target_worker.execute_model.assert_called_once_with(
441
+ **execute_model_data.to_dict())
442
+
443
+
444
+@pytest.mark.parametrize('k', [0, 5])
445
+@pytest.mark.parametrize('batch_size', [0])
446
+@torch.inference_mode()
447
+def test_empty_input_batch(k: int, batch_size: int):
448
+ """Verify that the SpecDecodeWorker calls the draft and target workers
449
+ when the input batch is empty. This can happen if the engine communicates
450
+ to the workers information without scheduling a batch.
451
+ """
452
+ draft_worker = mock_worker(cls=MultiStepWorker)
453
+ target_worker = mock_worker()
454
+ rejection_sampler = MagicMock(spec=RejectionSampler)
455
+ rejection_sampler.token_id_dtype = torch.int64
456
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
457
+
458
+ draft_worker.device = 'cuda'
459
+ target_worker.device = 'cuda'
460
+
461
+ set_random_seed(1)
462
+
463
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
464
+ metrics_collector)
465
+
466
+ execute_model_data, prompts, prev_output_tokens = create_batch(
467
+ batch_size, k, prev_output_token_len=0)
468
+
469
+ out = worker.execute_model(**execute_model_data.to_dict(),
470
+ num_spec_tokens=k)
471
+
472
+ assert len(out) == 1, f"expected only one token output when {k=}"
473
+ assert out[0].probs is None, "expect gpu tensor references to be None"
474
+ assert out[
475
+ 0].sampled_tokens is None, "expect gpu tensor references to be None"
476
+
477
+ draft_worker.execute_model.assert_called_once_with(
478
+ **execute_model_data.to_dict(), return_python_output=False)
479
+ target_worker.execute_model.assert_called_once_with(
480
+ **execute_model_data.to_dict())
481
+
482
+
483
+@torch.inference_mode()
484
+def test_init_model():
485
+ """Verify SpecDecodeWorker invokes proposer/scorer worker init_model, as
486
+ well as other GPU initialization.
487
+ """
488
+ draft_worker = mock_worker(cls=MultiStepWorker)
489
+ target_worker = mock_worker()
490
+ rejection_sampler = MagicMock(spec=RejectionSampler)
491
+ rejection_sampler.token_id_dtype = torch.int64
492
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
493
+
494
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
495
+ metrics_collector)
496
+
497
+ worker.init_model()
498
+
499
+ draft_worker.init_model.assert_called_once()
500
+
501
+ target_worker.init_model.assert_called_once()
502
+
503
+ metrics_collector.init_gpu_tensors.assert_called_once()
504
+ rejection_sampler.init_gpu_tensors.assert_called_once()
505
+
506
+
507
+@torch.inference_mode()
508
+def test_init_cache_engine():
509
+ """Verify SpecDecodeWorker invokes init_cache_engine on proposer/scorer
510
+ workers.
511
+ """
512
+ draft_worker = mock_worker(cls=MultiStepWorker)
513
+ target_worker = mock_worker()
514
+ rejection_sampler = MagicMock(spec=RejectionSampler)
515
+ rejection_sampler.token_id_dtype = torch.int64
516
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
517
+
518
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
519
+ metrics_collector)
520
+
521
+ cache_config = MagicMock()
522
+
523
+ worker.init_cache_engine(cache_config)
524
+
525
+ draft_worker.init_cache_engine.assert_called_once_with(cache_config)
526
+ target_worker.init_cache_engine.assert_called_once_with(cache_config)
527
+
528
+
529
+@pytest.mark.parametrize('available_gpu_blocks', [1, 1024])
530
+@pytest.mark.parametrize('available_cpu_blocks', [500])
531
+@pytest.mark.parametrize('target_cache_block_size_bytes', [2 * 2 * 4096])
532
+@pytest.mark.parametrize('draft_kv_size_bytes', [0, 2 * 2 * 768, 2 * 2 * 4096])
533
+@torch.inference_mode()
534
+def test_profile_num_available_blocks(available_gpu_blocks: int,
535
+ available_cpu_blocks: int,
536
+ target_cache_block_size_bytes: int,
537
+ draft_kv_size_bytes: int):
538
+ """Verify SpecDecodeWorker correctly profiles num available GPU blocks.
539
+ Specifically, it should run profiling in the scorer worker, and then evenly
540
+ split the blocks between proposer and scorer worker.
541
+ """
542
+ draft_worker = mock_worker(cls=MultiStepWorker)
543
+ target_worker = mock_worker()
544
+ rejection_sampler = MagicMock(spec=RejectionSampler)
545
+ rejection_sampler.token_id_dtype = torch.int64
546
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
547
+
548
+ target_worker.profile_num_available_blocks.return_value = (
549
+ available_gpu_blocks, available_cpu_blocks)
550
+ target_worker.get_cache_block_size_bytes.return_value = target_cache_block_size_bytes
551
+ draft_worker.get_cache_block_size_bytes.return_value = draft_kv_size_bytes
552
+
553
+ worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
554
+ metrics_collector)
555
+
556
+ # These values do not directly impact the adjusted block size calculation,
557
+ # so they can be fixed.
558
+ gpu_memory_utilization = 0.9
559
+ cpu_swap_space = 100
560
+ block_size = 16
561
+
562
+ num_gpu_blocks, num_cpu_blocks = worker.profile_num_available_blocks(
563
+ block_size, gpu_memory_utilization, cpu_swap_space, cache_dtype="auto")
564
+
565
+ target_worker.profile_num_available_blocks.assert_called_once_with(
566
+ block_size, gpu_memory_utilization, cpu_swap_space, "auto")
567
+ assert num_cpu_blocks == available_cpu_blocks
568
+
569
+ assert num_gpu_blocks == split_num_cache_blocks_evenly(
570
+ target_cache_block_size_bytes, draft_kv_size_bytes,
571
+ available_gpu_blocks)
572
+
573
+
574
+@pytest.mark.parametrize('available_gpu_blocks',
575
+ list(range(20)) + [1024, 1024**2])
576
+@pytest.mark.parametrize('target_cache_block_size_bytes',
577
+ [2 * 2 * 4096, 2 * 2 * 8192])
578
+@pytest.mark.parametrize('draft_kv_size_bytes', [0, 2 * 2 * 768, 2 * 2 * 4096])
579
+@torch.inference_mode()
580
+def test_split_num_cache_blocks_evenly(available_gpu_blocks: int,
581
+ target_cache_block_size_bytes: int,
582
+ draft_kv_size_bytes: int):
583
+ """Verify split_num_cache_blocks_evenly does not exceed original memory
584
+ allocation in bytes.
585
+ """
586
+ num_blocks = split_num_cache_blocks_evenly(target_cache_block_size_bytes,
587
+ draft_kv_size_bytes,
588
+ available_gpu_blocks)
589
+ assert (num_blocks * target_cache_block_size_bytes) + (
590
+ num_blocks * draft_kv_size_bytes) <= (available_gpu_blocks *
591
+ target_cache_block_size_bytes)
tests/spec_decode/test_utils.py
new
+111
@@ -0,0 +1,111 @@
1
+from vllm.spec_decode.util import get_all_seq_ids
2
+from vllm.sequence import SequenceGroupMetadata
3
+from vllm.spec_decode.util import split_batch_by_proposal_len
4
+
5
+import pytest
6
+from unittest.mock import MagicMock
7
+
8
+
9
+def test_get_all_seq_ids():
10
+ """Verify get_all_seq_ids extracts all seq ids.
11
+ """
12
+ expected_seq_ids = list(range(10)) + list(range(100, 110))
13
+
14
+ seq_group_metadata_list = [
15
+ SequenceGroupMetadata(
16
+ request_id=str(seq_id),
17
+ is_prompt=True,
18
+ seq_data={
19
+ seq_id: MagicMock(),
20
+ },
21
+ sampling_params=MagicMock(),
22
+ block_tables={
23
+ seq_id: MagicMock(),
24
+ },
25
+ lora_request=None,
26
+ ) for seq_id in expected_seq_ids
27
+ ]
28
+
29
+ actual_seq_ids = get_all_seq_ids(seq_group_metadata_list)
30
+ assert actual_seq_ids == expected_seq_ids
31
+
32
+
33
+@pytest.fixture
34
+def fake_sequence_group_metadata():
35
+ seq_ids = list(range(3))
36
+ return [
37
+ SequenceGroupMetadata(
38
+ request_id=str(i),
39
+ is_prompt=True,
40
+ seq_data={
41
+ i: MagicMock(),
42
+ },
43
+ sampling_params=MagicMock(),
44
+ block_tables={
45
+ i: MagicMock(),
46
+ },
47
+ lora_request=None,
48
+ ) for i in seq_ids
49
+ ]
50
+
51
+
52
+def test_filter_zero_length_proposals(fake_sequence_group_metadata):
53
+ proposal_lens = [0, 1, 0]
54
+ filtered_groups, indices = split_batch_by_proposal_len(
55
+ fake_sequence_group_metadata,
56
+ proposal_lens,
57
+ select_proposal_len_zero=True)
58
+
59
+ expected_groups = [
60
+ fake_sequence_group_metadata[0], fake_sequence_group_metadata[2]
61
+ ]
62
+ expected_indices = [0, 2]
63
+
64
+ assert filtered_groups == expected_groups
65
+ assert indices == expected_indices
66
+
67
+
68
+def test_filter_non_zero_length_proposals(fake_sequence_group_metadata):
69
+ proposal_lens = [0, 1, 2]
70
+ filtered_groups, indices = split_batch_by_proposal_len(
71
+ fake_sequence_group_metadata,
72
+ proposal_lens,
73
+ select_proposal_len_zero=False)
74
+
75
+ expected_groups = [
76
+ fake_sequence_group_metadata[1], fake_sequence_group_metadata[2]
77
+ ]
78
+ expected_indices = [1, 2]
79
+
80
+ assert filtered_groups == expected_groups
81
+ assert indices == expected_indices
82
+
83
+
84
+def test_empty_inputs():
85
+ filtered_groups, indices = split_batch_by_proposal_len(
86
+ [], [], select_proposal_len_zero=True)
87
+
88
+ assert filtered_groups == []
89
+ assert indices == []
90
+
91
+
92
+def test_all_zero_with_non_zero_filter(fake_sequence_group_metadata):
93
+ proposal_lens = [0, 0, 0]
94
+ filtered_groups, indices = split_batch_by_proposal_len(
95
+ fake_sequence_group_metadata,
96
+ proposal_lens,
97
+ select_proposal_len_zero=False)
98
+
99
+ assert filtered_groups == []
100
+ assert indices == []
101
+
102
+
103
+def test_all_non_zero_with_zero_filter(fake_sequence_group_metadata):
104
+ proposal_lens = [1, 1, 1]
105
+ filtered_groups, indices = split_batch_by_proposal_len(
106
+ fake_sequence_group_metadata,
107
+ proposal_lens,
108
+ select_proposal_len_zero=True)
109
+
110
+ assert filtered_groups == []
111
+ assert indices == []
tests/spec_decode/utils.py
renamed
+96
-19
@@ -1,13 +1,16 @@
1
import torch
2
-from typing import List, Optional, Dict
2
+from typing import List, Optional, Dict, Iterable, Union
3
+from unittest.mock import MagicMock
4
5
from vllm.worker.worker import Worker
6
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7
from vllm.engine.arg_utils import EngineArgs
7
-from vllm.sequence import Logprob, SequenceGroupMetadata, SequenceData
8
+from vllm.sequence import (Logprob, SequenceGroupMetadata, SequenceData,
9
+ SamplerOutput, SequenceGroupOutput, SequenceOutput)
10
from vllm.sampling_params import SamplingParams
11
from vllm.worker.cache_engine import CacheEngine
12
from vllm.model_executor.utils import set_random_seed
13
+from itertools import count
14
from dataclasses import dataclass, fields
15
16
@@ -24,6 +27,11 @@ class ExecuteModelData:
27
return dict(
28
(field.name, getattr(self, field.name)) for field in fields(self))
29
30
+ @classmethod
31
+ def from_dict(cls, d):
32
+ cleaned = dict((field.name, d[field.name]) for field in fields(cls))
33
+ return cls(**cleaned)
34
+
35
36
def round_up_to_next_block(seq_len: int, block_size: int) -> int:
37
return (seq_len + block_size - 1) // block_size
@@ -50,6 +58,21 @@ def create_execute_model_data(
58
)
59
60
61
+def mock_worker(cls=None,
62
+ vocab_size: int = 30_000,
63
+ max_model_len: int = 2048,
64
+ rank: int = 0) -> MagicMock:
65
+ if cls is None:
66
+ cls = Worker
67
+
68
+ worker = MagicMock(spec=cls)
69
+ worker.vocab_size = vocab_size
70
+ worker.max_model_len = max_model_len
71
+ worker.rank = rank
72
+ worker.device = 'cuda:0'
73
+ return worker
74
+
75
+
76
def patch_execute_model_with_seeds(worker: Worker, rand_seeds: List[int]):
77
seed_iter = iter(rand_seeds)
78
original_execute_model = worker.execute_model
@@ -117,25 +140,12 @@ def create_seq_group_metadata_from_prompts(
140
block_size: int,
141
final_seq_lens: List[int],
142
continuations: Optional[List[List[int]]] = None,
120
- num_tokens_processed: Optional[List[int]] = None,
143
seq_ids: Optional[List[int]] = None,
144
) -> List[SequenceGroupMetadata]:
145
146
if continuations is None:
147
continuations = [[] for _ in prompts]
148
127
- if num_tokens_processed is None:
128
- # Default to 1 token missing from kv cache for generation sequences.
129
- num_tokens_processed = []
130
- for continuation, prompt in zip(continuations, prompts):
131
- # If prefill, then default to zero tokens processed.
132
- if not continuation:
133
- num_tokens_processed.append(0)
134
- else:
135
- # If generation, then default to all but one tokens processed.
136
- num_tokens_processed.append(
137
- len(continuation) + len(prompt) - 1)
138
-
149
if seq_ids is None:
150
seq_ids = list(i for i, _ in enumerate(prompts))
151
@@ -155,13 +165,15 @@ def create_seq_group_metadata_from_prompts(
165
is_prompt=len(cont_token_ids) == 0,
166
seq_data={
167
i:
158
- SequenceData(prompt_token_ids=prompt_token_ids[:] +
159
- cont_token_ids[:])
168
+ SequenceData(
169
+ prompt_token_ids=prompt_token_ids[:],
170
+ output_token_ids=cont_token_ids[:],
171
+ ),
172
},
173
sampling_params=SamplingParams(temperature=0.0, ),
174
block_tables={i: block_allocations[i][:]},
163
- ) for i, (prompt_token_ids, cont_token_ids, num_tokens_saved) in
164
- enumerate(zip(prompts, continuations, num_tokens_processed))
175
+ ) for i, (prompt_token_ids,
176
+ cont_token_ids) in enumerate(zip(prompts, continuations))
177
]
178
179
@@ -178,3 +190,68 @@ def assert_logprobs_dict_allclose(
190
expected = torch.tensor(
191
single_step_expected_logprobs[token_id].logprob)
192
assert torch.allclose(actual, expected)
193
+
194
+
195
+def create_sampler_output_list(
196
+ token_ids: torch.Tensor,
197
+ probs: Iterable[Optional[torch.Tensor]],
198
+ seq_ids: Optional[List[int]] = None) -> List[SamplerOutput]:
199
+ num_steps, batch_size = token_ids.shape
200
+ token_ids_by_step = token_ids.tolist()
201
+
202
+ if seq_ids is None:
203
+ seq_ids = list(range(batch_size))
204
+
205
+ return [
206
+ SamplerOutput(outputs=[
207
+ SequenceGroupOutput(
208
+ samples=[
209
+ SequenceOutput(
210
+ output_token=token_id,
211
+ parent_seq_id=seq_ids[seq_index],
212
+ logprobs={token_id: 0},
213
+ )
214
+ ],
215
+ prompt_logprobs=None,
216
+ ) for seq_index, token_id in enumerate(token_ids_by_step[step])
217
+ ],
218
+ sampled_token_probs=probs[step],
219
+ sampled_token_ids=token_ids[step])
220
+ for step in range(num_steps)
221
+ ]
222
+
223
+
224
+def create_batch(batch_size,
225
+ k,
226
+ prompt_len: Union[int, List[int]] = 10,
227
+ prev_output_token_len: int = 10,
228
+ seq_ids: Optional[List[int]] = None,
229
+ num_gpu_blocks: Optional[int] = None,
230
+ block_size: Optional[int] = None):
231
+ if block_size is None:
232
+ block_size = 8
233
+
234
+ if num_gpu_blocks is None:
235
+ num_gpu_blocks = 2048 // block_size
236
+
237
+ iterator = count()
238
+
239
+ if isinstance(prompt_len, int):
240
+ prompt_lens = [prompt_len for _ in range(batch_size)]
241
+ else:
242
+ prompt_lens = prompt_len
243
+
244
+ prompts = [[next(iterator) for _ in range(p_len)] for p_len in prompt_lens]
245
+ prev_output_tokens = [[
246
+ next(iterator) for _ in range(prev_output_token_len)
247
+ ] for _ in range(batch_size)]
248
+ final_seq_lens = [
249
+ len(prompt) + len(prev_output_token) + k + 1
250
+ for prompt, prev_output_token in zip(prompts, prev_output_tokens)
251
+ ]
252
+
253
+ execute_model_data = create_execute_model_data(
254
+ create_seq_group_metadata_from_prompts(prompts, num_gpu_blocks,
255
+ block_size, final_seq_lens,
256
+ prev_output_tokens, seq_ids), )
257
+ return execute_model_data, prompts, prev_output_tokens
tests/test_sequence.py
new
+50
@@ -0,0 +1,50 @@
1
+import pytest
2
+
3
+from vllm.sequence import SequenceGroupOutput, SamplerOutput, SequenceOutput
4
+
5
+
6
+@pytest.fixture
7
+def sample_outputs():
8
+ return [
9
+ SequenceGroupOutput(samples=[
10
+ SequenceOutput(parent_seq_id=0, output_token=i, logprobs={})
11
+ ],
12
+ prompt_logprobs=None) for i in range(5)
13
+ ]
14
+
15
+
16
+@pytest.fixture
17
+def sampler_output(sample_outputs):
18
+ return SamplerOutput(outputs=sample_outputs)
19
+
20
+
21
+def test_sampler_output_initialization(sampler_output, sample_outputs):
22
+ assert len(sampler_output) == len(sample_outputs)
23
+ assert sampler_output.sampled_token_probs is None
24
+ assert sampler_output.sampled_token_ids is None
25
+ assert sampler_output.spec_decode_worker_metrics is None
26
+
27
+
28
+def test_sampler_output_getitem(sampler_output, sample_outputs):
29
+ assert sampler_output[2] == sample_outputs[2]
30
+
31
+
32
+def test_sampler_output_setitem(sampler_output):
33
+ new_output = SequenceGroupOutput(samples=[
34
+ SequenceOutput(parent_seq_id=0, output_token=99, logprobs={})
35
+ ],
36
+ prompt_logprobs=None)
37
+ sampler_output[2] = new_output
38
+ assert sampler_output[2] == new_output
39
+
40
+
41
+def test_sampler_output_len(sampler_output, sample_outputs):
42
+ assert len(sampler_output) == len(sample_outputs)
43
+
44
+
45
+def test_sampler_output_eq(sample_outputs):
46
+ sampler_output1 = SamplerOutput(outputs=sample_outputs)
47
+ sampler_output2 = SamplerOutput(outputs=sample_outputs.copy())
48
+ sampler_output3 = SamplerOutput(outputs=sample_outputs[:-1])
49
+ assert sampler_output1 == sampler_output2
50
+ assert sampler_output1 != sampler_output3
vllm/model_executor/layers/rejection_sampler.py
+8
-2
@@ -21,8 +21,6 @@ class RejectionSampler(nn.Module):
21
nontrivial latency.
22
"""
23
super().__init__()
24
- self.probs_dtype = torch.float32
25
- self.token_id_dtype = torch.int64
24
self._strict_mode = strict_mode
25
26
# NOTE: A "bonus token" is accepted iff all proposal tokens are
@@ -44,6 +42,14 @@ class RejectionSampler(nn.Module):
42
dtype=torch.long,
43
device=device)
44
45
+ @property
46
+ def probs_dtype(self):
47
+ return torch.float32
48
+
49
+ @property
50
+ def token_id_dtype(self):
51
+ return torch.int64
52
+
53
def forward(
54
self,
55
target_probs: torch.Tensor,
vllm/model_executor/layers/sampler.py
+1
-1
@@ -587,4 +587,4 @@ def _build_sampler_output(
587
SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
588
sampler_output.append(
589
SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
590
- return sampler_output
590
+ return SamplerOutput(outputs=sampler_output)
vllm/sequence.py
+50
-5
@@ -2,12 +2,16 @@
2
import copy
3
import enum
4
from dataclasses import dataclass
5
-from typing import Dict, List, Optional, Union
5
+from typing import Dict, List, Optional, Union, TYPE_CHECKING
6
7
from vllm.block import LogicalTokenBlock
8
from vllm.sampling_params import SamplingParams
9
from vllm.lora.request import LoRARequest
10
11
+if TYPE_CHECKING:
12
+ import torch
13
+ from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
14
+
15
16
@dataclass
17
class Logprob:
@@ -81,6 +85,8 @@ class SequenceData:
85
86
Args:
87
prompt_token_ids: The token IDs of the prompt.
88
+ output_token_ids: The token IDs of the output. Set to an empty list if
89
+ None.
90
91
Attributes:
92
prompt_token_ids: The token IDs of the prompt.
@@ -91,9 +97,13 @@ class SequenceData:
97
def __init__(
98
self,
99
prompt_token_ids: List[int],
100
+ output_token_ids: Optional[List[int]] = None,
101
) -> None:
102
+ if output_token_ids is None:
103
+ output_token_ids = []
104
+
105
self.prompt_token_ids = prompt_token_ids
96
- self.output_token_ids: List[int] = []
106
+ self.output_token_ids = output_token_ids
107
self.cumulative_logprob = 0.0
108
109
def append_token_id(self, token_id: int, logprob: float) -> None:
@@ -117,6 +127,12 @@ class SequenceData:
127
return self.prompt_token_ids[-1]
128
return self.output_token_ids[-1]
129
130
+ def get_prompt_token_ids(self) -> int:
131
+ return self.prompt_token_ids
132
+
133
+ def get_output_token_ids(self) -> int:
134
+ return self.output_token_ids
135
+
136
def __repr__(self) -> str:
137
return (f"SequenceData("
138
f"prompt_token_ids={self.prompt_token_ids}, "
@@ -506,6 +522,35 @@ class SequenceGroupOutput:
522
and self.prompt_logprobs == other.prompt_logprobs)
523
524
509
-# For each sequence group, we generate a list of SequenceOutput object,
510
-# each of which contains one possible candidate for the next token.
511
-SamplerOutput = List[SequenceGroupOutput]
525
+@dataclass
526
+class SamplerOutput:
527
+ """For each sequence group, we generate a list of SequenceOutput object,
528
+ each of which contains one possible candidate for the next token.
529
+
530
+ This datastructure implements methods so it can be used like a list, but
531
+ also has optional fields for device tensors.
532
+ """
533
+
534
+ outputs: List[SequenceGroupOutput]
535
+
536
+ # On-device tensor containing probabilities of each token.
537
+ sampled_token_probs: Optional["torch.Tensor"] = None
538
+
539
+ # On-device tensor containing the sampled token ids.
540
+ sampled_token_ids: Optional["torch.Tensor"] = None
541
+
542
+ # Spec decode metrics populated by workers.
543
+ spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
544
+
545
+ def __getitem__(self, idx: int):
546
+ return self.outputs[idx]
547
+
548
+ def __setitem__(self, idx: int, value):
549
+ self.outputs[idx] = value
550
+
551
+ def __len__(self):
552
+ return len(self.outputs)
553
+
554
+ def __eq__(self, other: object):
555
+ return isinstance(other,
556
+ self.__class__) and self.outputs == other.outputs
vllm/spec_decode/batch_expansion.py
new
+351
@@ -0,0 +1,351 @@
1
+from typing import Iterator, List, Tuple, Optional, Dict
2
+from itertools import chain, count
3
+
4
+import torch
5
+
6
+from vllm.sequence import (SamplerOutput, SequenceGroupMetadata, SequenceData)
7
+from vllm.worker.worker import Worker
8
+from vllm.spec_decode.util import nvtx_range, sampler_output_to_torch, get_all_seq_ids, split_batch_by_proposal_len
9
+from vllm.spec_decode.interfaces import SpeculativeScorer, SpeculativeProposals, SpeculativeScores
10
+
11
+SeqId = int
12
+TargetSeqId = int
13
+TokenId = int
14
+
15
+
16
+class BatchExpansionTop1Scorer(SpeculativeScorer):
17
+ """Implements a speculative scorer that uses batch expansion to get
18
+ probabilities of speculative tokens according to the scoring model.
19
+
20
+ Batch expansion converts a list of sequences and multiple query positions
21
+ to a new batch of sequences, each with a single query position. This allows
22
+ for MQA-like scoring in speculative decoding without requiring an MQA
23
+ kernel.
24
+
25
+ It is strictly less efficient than MQA scoring.
26
+
27
+ It only supports scoring the top1 proposal tokens of the proposer, instead
28
+ of topk/tree.
29
+ """
30
+
31
+ def __init__(self, scorer_worker: Worker, device: str, vocab_size: int):
32
+ self._scorer_worker = scorer_worker
33
+ self._device = device
34
+ self._vocab_size = vocab_size
35
+
36
+ @nvtx_range("BatchExpansionTop1Scorer.score_proposals")
37
+ def score_proposals(
38
+ self,
39
+ seq_group_metadata_list: List[SequenceGroupMetadata],
40
+ blocks_to_swap_in: Optional[Dict[int, int]],
41
+ blocks_to_swap_out: Optional[Dict[int, int]],
42
+ blocks_to_copy: Optional[Dict[int, List[int]]],
43
+ k: int,
44
+ proposals: SpeculativeProposals,
45
+ ) -> SpeculativeScores:
46
+ """Score the proposed tokens via the scorer model.
47
+
48
+ This converts each input sequence to a set of k+1 target sequences. The
49
+ target sequences have the unique continuations to be scored and a
50
+ unique sequence ID that is different from all input sequence ids.
51
+
52
+ If a speculative sequence length would exceed the max model length, then
53
+ no speculation is produced for that sequence.
54
+
55
+ Args:
56
+ seq_group_metadata_list: The input sequence group metadata.
57
+ blocks_to_swap_in: This is passed to the worker during scoring.
58
+ blocks_to_swap_out: This is passed to the worker during scoring.
59
+ blocks_to_copy: This is passed to the worker during scoring.
60
+ k: The fixed proposal length.
61
+ proposals: The speculative proposals to score.
62
+ Returns:
63
+ SpeculativeScores: The scores of each speculative token, along with
64
+ which sequences were ignored during scoring.
65
+ """
66
+
67
+ # TODO(cade) perform this on GPU to remove blocking call.
68
+ proposal_lens_list = proposals.proposal_lens.tolist()
69
+ proposal_token_ids_list = proposals.proposal_token_ids.tolist()
70
+
71
+ spec_indices, non_spec_indices, target_seq_group_metadata_list, num_scoring_tokens = self._expand_batch(
72
+ seq_group_metadata_list=seq_group_metadata_list,
73
+ proposal_token_ids_list=proposal_token_ids_list,
74
+ proposal_lens_list=proposal_lens_list,
75
+ )
76
+
77
+ target_sampler_output = self._scorer_worker.execute_model(
78
+ seq_group_metadata_list=target_seq_group_metadata_list,
79
+ blocks_to_swap_in=blocks_to_swap_in,
80
+ blocks_to_swap_out=blocks_to_swap_out,
81
+ blocks_to_copy=blocks_to_copy,
82
+ return_python_output=False)
83
+
84
+ all_tokens, all_probs = self._contract_batch(
85
+ original_bs=len(seq_group_metadata_list),
86
+ target_sampler_output=target_sampler_output,
87
+ proposals=proposals,
88
+ num_scoring_tokens=num_scoring_tokens,
89
+ non_spec_indices=non_spec_indices,
90
+ spec_indices=spec_indices,
91
+ k=k,
92
+ )
93
+
94
+ return SpeculativeScores(
95
+ probs=all_probs,
96
+ token_ids=all_tokens,
97
+ )
98
+
99
+ def _expand_batch(
100
+ self,
101
+ seq_group_metadata_list: List[SequenceGroupMetadata],
102
+ proposal_token_ids_list: List[TokenId],
103
+ proposal_lens_list: List[int],
104
+ ) -> Tuple[List[int], List[int], List[SequenceGroupMetadata], int]:
105
+ """Given the input sequences and potentially multiple corresponding
106
+ proposal tokens, create a new batch where each sequence has a single
107
+ query token.
108
+ """
109
+
110
+ # vLLM currently only supports proposal lens equal to zero or the batch
111
+ # proposal len. This adds some complexity (splitting the batch into spec
112
+ # and non spec sequences) and should be removed in the future. It can be
113
+ # done by supporting per-sequence proposal lens.
114
+ spec_seqs, spec_indices = split_batch_by_proposal_len(
115
+ seq_group_metadata_list,
116
+ proposal_lens_list,
117
+ select_proposal_len_zero=False)
118
+ non_spec_seqs, non_spec_indices = split_batch_by_proposal_len(
119
+ seq_group_metadata_list,
120
+ proposal_lens_list,
121
+ select_proposal_len_zero=True)
122
+
123
+ target_seq_group_metadata_list = self._create_scoring_model_input(
124
+ spec_seqs, proposal_token_ids_list)
125
+ num_scoring_tokens = len(target_seq_group_metadata_list)
126
+ target_seq_group_metadata_list.extend(non_spec_seqs)
127
+
128
+ return spec_indices, non_spec_indices, target_seq_group_metadata_list, num_scoring_tokens
129
+
130
+ def _contract_batch(self, original_bs: int,
131
+ target_sampler_output: List[SamplerOutput],
132
+ proposals: SpeculativeProposals,
133
+ num_scoring_tokens: int, non_spec_indices: List[int],
134
+ spec_indices: List[int],
135
+ k: int) -> Tuple[torch.Tensor, torch.Tensor]:
136
+ """Contract the expanded batch back into its original size.
137
+ This maps the scores of speculative tokens back to their original
138
+ sequences.
139
+ """
140
+ (target_token_ids, target_probs, non_spec_target_token_ids,
141
+ non_spec_target_probs) = self._split_scoring_output(
142
+ target_sampler_output, num_scoring_tokens)
143
+
144
+ # Map distinct sequences used to score each token
145
+ # of shape [batch_size * k + 1] back to [batch_size, k + 1].
146
+ batch_size, k = proposals.proposal_token_ids.shape
147
+
148
+ target_token_ids = target_token_ids.squeeze().reshape(
149
+ batch_size, k + 1)
150
+ target_probs = target_probs.squeeze().reshape(batch_size, k + 1,
151
+ self._vocab_size)
152
+
153
+ all_tokens = torch.full(size=(original_bs, k + 1),
154
+ fill_value=-1,
155
+ device=self._device,
156
+ dtype=torch.long)
157
+ all_probs = torch.zeros(original_bs,
158
+ k + 1,
159
+ self._vocab_size,
160
+ device=self._device,
161
+ dtype=torch.float32)
162
+
163
+ if non_spec_indices:
164
+ all_tokens[non_spec_indices, 0] = non_spec_target_token_ids
165
+ all_probs[non_spec_indices, :1, :] = non_spec_target_probs
166
+
167
+ if spec_indices:
168
+ all_tokens[spec_indices] = target_token_ids
169
+ all_probs[spec_indices] = target_probs
170
+
171
+ return all_tokens, all_probs
172
+
173
+ def _create_scoring_model_input(
174
+ self,
175
+ seq_group_metadata_list: List[SequenceGroupMetadata],
176
+ proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k]
177
+ ) -> List[SequenceGroupMetadata]:
178
+ """Given the original input sequences and proposed tokens from the draft
179
+ model, create a list of target sequences that can be used for scoring.
180
+ """
181
+
182
+ if not seq_group_metadata_list:
183
+ return []
184
+
185
+ target_seq_ids_iter = self._create_target_seq_id_iterator(
186
+ get_all_seq_ids(seq_group_metadata_list))
187
+
188
+ target_seq_group_metadata = list(
189
+ chain.from_iterable(
190
+ self._create_target_seq_group_metadata(
191
+ seq_group_metadata,
192
+ proposal_token_ids,
193
+ i,
194
+ target_seq_ids_iter,
195
+ ) for i, seq_group_metadata in enumerate(
196
+ seq_group_metadata_list)))
197
+
198
+ return target_seq_group_metadata
199
+
200
+ def _create_target_seq_group_metadata(
201
+ self,
202
+ input_seq_group_metadata: SequenceGroupMetadata,
203
+ proposal_token_ids: List[TokenId], # shape: [batch_size, k]
204
+ batch_index: int,
205
+ target_seq_ids_iter: Iterator[TargetSeqId],
206
+ ) -> List[SequenceGroupMetadata]:
207
+ """Given an input sequence group metadata and a list of draft tokens,
208
+ create a list of target SequenceGroupMetadata, one for each
209
+ token id that needs to be scored.
210
+
211
+ Naive speculative decoding requires K target model scores, one for each
212
+ draft model token. However one can add a bonus token such that if each
213
+ token is accepted, then a final token may be sampled from the model.
214
+ This function creates K+1 target SequenceGroupMetadata to take
215
+ advantage of the bonus token.
216
+ """
217
+ assert not input_seq_group_metadata.is_prompt, (
218
+ "Speculating on "
219
+ "prompts not yet supported")
220
+ assert len(input_seq_group_metadata.seq_data) == 1, (
221
+ "Beam search "
222
+ "not supported in speculative decoding")
223
+ input_seq_id = next(iter(input_seq_group_metadata.seq_data.keys()))
224
+
225
+ token_ids_to_score = self._get_token_ids_to_score(
226
+ proposal_token_ids[batch_index])
227
+
228
+ target_seq_group_metadata_list: List[SequenceGroupMetadata] = []
229
+ for token_ids in token_ids_to_score:
230
+ target_seq_group_metadata_list.append(
231
+ self._create_single_target_seq_group_metadata(
232
+ input_seq_group_metadata,
233
+ input_seq_id,
234
+ next(target_seq_ids_iter),
235
+ token_ids,
236
+ ))
237
+
238
+ return target_seq_group_metadata_list
239
+
240
+ def _create_single_target_seq_group_metadata(
241
+ self,
242
+ seq_group_metadata: SequenceGroupMetadata,
243
+ seq_id: SeqId,
244
+ target_seq_id: TargetSeqId,
245
+ token_ids: List[TokenId],
246
+ ) -> SequenceGroupMetadata:
247
+ """Create a single target SequenceGroupMetadata.
248
+
249
+ Args:
250
+ seq_group_metadata: The metadata for the input sequence.
251
+ seq_id: The input sequence ID.
252
+ target_seq_id: The corresponding target sequence ID.
253
+ token_ids: The list of token ids that are to be appended to the
254
+ input sequence.
255
+ """
256
+ seq_data = seq_group_metadata.seq_data[seq_id]
257
+ prompt_token_ids = seq_data.get_prompt_token_ids()
258
+ new_output_token_ids = [*seq_data.get_output_token_ids(), *token_ids]
259
+
260
+ return SequenceGroupMetadata(
261
+ request_id=seq_group_metadata.request_id,
262
+ is_prompt=seq_group_metadata.is_prompt,
263
+ seq_data={
264
+ target_seq_id:
265
+ SequenceData(
266
+ prompt_token_ids=prompt_token_ids,
267
+ output_token_ids=new_output_token_ids,
268
+ ),
269
+ },
270
+ sampling_params=seq_group_metadata.sampling_params,
271
+ block_tables={
272
+ target_seq_id: seq_group_metadata.block_tables[seq_id],
273
+ },
274
+ lora_request=None,
275
+ )
276
+
277
+ def _split_scoring_output(
278
+ self, sampler_output: SamplerOutput, num_scoring_tokens: int
279
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
280
+ """Split the target model output into speculative and non-speculative
281
+ output.
282
+ """
283
+
284
+ # vLLM currently only supports proposal lens equal to zero or the batch
285
+ # proposal len. This adds some complexity (splitting the batch into spec
286
+ # and non spec sequences) and should be removed in the future. It can be
287
+ # done by supporting per-sequence proposal lens.
288
+ #
289
+ # First samples are from speculative scoring, latter samples are non-
290
+ # speculative samples.
291
+ split_sizes = [
292
+ num_scoring_tokens,
293
+ sampler_output.sampled_token_ids.numel() - num_scoring_tokens
294
+ ]
295
+ (spec_probs, non_spec_probs
296
+ ) = sampler_output.sampled_token_probs.split(split_sizes)
297
+ (spec_sampled_tokens, non_spec_sampled_tokens
298
+ ) = sampler_output.sampled_token_ids.flatten().split(split_sizes)
299
+
300
+ # Convert scores to tensors.
301
+ sampler_output.sampled_token_probs = spec_probs
302
+ sampler_output.sampled_token_ids = spec_sampled_tokens
303
+ target_token_ids, target_probs = sampler_output_to_torch(
304
+ [sampler_output])
305
+
306
+ # Convert non-speculative output tokens to tensors.
307
+ sampler_output.sampled_token_probs = non_spec_probs
308
+ sampler_output.sampled_token_ids = non_spec_sampled_tokens
309
+ non_spec_target_token_ids, non_spec_target_probs = sampler_output_to_torch(
310
+ [sampler_output])
311
+
312
+ return target_token_ids, target_probs, non_spec_target_token_ids, non_spec_target_probs
313
+
314
+ def _create_target_seq_id_iterator(
315
+ self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
316
+ """Create an iterator for creating target sequence ids.
317
+ Target sequence ids are distinct from sequence ids because we create a
318
+ distinct target sequence id for each proposal token to be scored.
319
+
320
+ This implementation increments a counter starting at 1 + max of all
321
+ provided input sequence ids.
322
+ """
323
+ return count(start=max(seq_ids) + 1)
324
+
325
+ def _get_token_ids_to_score(
326
+ self,
327
+ full_spec_token_ids: List[TokenId] # shape: [k]
328
+ ) -> List[List[TokenId]]:
329
+ """Given an int tensor of proposal token ids, return a list of
330
+ token ids that should be scored.
331
+
332
+ Returns k+1 output lists. The additional one is used for generating the
333
+ bonus token.
334
+
335
+ Example:
336
+ Input: [0, 1, 2, 3] (k=4)
337
+ Output: (k+1 lists)
338
+ []
339
+ [0]
340
+ [0, 1]
341
+ [0, 1, 2]
342
+ [0, 1, 2, 3]
343
+ """
344
+ empty_token_ids = []
345
+
346
+ token_ids_to_score = [empty_token_ids]
347
+ token_ids_to_score.extend([
348
+ full_spec_token_ids[:i + 1]
349
+ for i in range(len(full_spec_token_ids))
350
+ ])
351
+ return token_ids_to_score
vllm/spec_decode/interfaces.py
new
+77
@@ -0,0 +1,77 @@
1
+from typing import List, Tuple, Optional, Dict
2
+from dataclasses import dataclass
3
+from abc import ABC, abstractmethod
4
+
5
+import torch
6
+
7
+from vllm.sequence import SequenceGroupMetadata
8
+
9
+
10
+@dataclass
11
+class SpeculativeProposals:
12
+ """Datastructure used to represent proposal tokens from some proposer. It
13
+ also tracks how many speculative tokens each sequence has.
14
+ """
15
+
16
+ # Speculative proposal tokens.
17
+ proposal_token_ids: torch.Tensor
18
+
19
+ # Probabilities of the proposal tokens according to the proposer.
20
+ proposal_probs: torch.Tensor
21
+
22
+ # The valid length of each proposal; can be zero.
23
+ proposal_lens: torch.Tensor
24
+
25
+ def __repr__(self):
26
+ return (f"SpeculativeProposals("
27
+ f"proposal_token_ids={self.proposal_token_ids.shape}, "
28
+ f"proposal_probs={self.proposal_probs.shape}, "
29
+ f"proposal_lens={self.proposal_lens.shape})")
30
+
31
+
32
+@dataclass
33
+class SpeculativeScores:
34
+ """Datastructure used to represent the scores of speculative tokens
35
+ according to the scoring model.
36
+ """
37
+
38
+ # Probabilities of the speculative tokens according to the scoring model.
39
+ probs: torch.Tensor
40
+
41
+ # Token ids sampled from the scoring model. Used for speculative bonus
42
+ # tokens and also non-speculative normal decoding.
43
+ token_ids: torch.Tensor
44
+
45
+ def __repr__(self):
46
+ return (f"SpeculativeScores("
47
+ f"probs={self.probs.shape}, "
48
+ f"token_ids={self.token_ids.shape})")
49
+
50
+
51
+class SpeculativeProposer(ABC):
52
+
53
+ @abstractmethod
54
+ def get_proposals(
55
+ self,
56
+ seq_group_metadata_list: List[SequenceGroupMetadata],
57
+ blocks_to_swap_in: Dict[int, int],
58
+ blocks_to_swap_out: Dict[int, int],
59
+ blocks_to_copy: Dict[int, List[int]],
60
+ max_proposal_len: int,
61
+ ) -> SpeculativeProposals:
62
+ raise NotImplementedError
63
+
64
+
65
+class SpeculativeScorer(ABC):
66
+
67
+ @abstractmethod
68
+ def score_proposals(
69
+ self,
70
+ seq_group_metadata_list: List[SequenceGroupMetadata],
71
+ blocks_to_swap_in: Optional[Dict[int, int]],
72
+ blocks_to_swap_out: Optional[Dict[int, int]],
73
+ blocks_to_copy: Optional[Dict[int, List[int]]],
74
+ k: int,
75
+ proposals: SpeculativeProposals,
76
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
77
+ raise NotImplementedError
vllm/spec_decode/metrics.py
new
+174
@@ -0,0 +1,174 @@
1
+import torch
2
+from dataclasses import dataclass
3
+from vllm.model_executor.layers.rejection_sampler import RejectionSampler
4
+from typing import Optional
5
+from vllm.utils import in_wsl
6
+import time
7
+from typing import Callable
8
+
9
+
10
+@dataclass
11
+class SpecDecodeWorkerMetrics:
12
+ """Dataclass holding metrics emitted from the spec decode worker.
13
+ """
14
+
15
+ # The empirical acceptance rate of the proposal method on a per-token basis.
16
+ # This is useful for evaluating how well the proposal method aligns with the
17
+ # scoring method.
18
+ draft_acceptance_rate: float
19
+
20
+ # The empirical efficiency, measured as the number of tokens emitted by the
21
+ # system divided by the number of tokens that could be emitted by the system
22
+ # if the proposal method were perfect.
23
+ system_efficiency: float
24
+
25
+ # The number of speculative tokens produced by the proposal method.
26
+ draft_tokens: int
27
+
28
+ # The number of tokens emitted by the entire system.
29
+ emitted_tokens: int
30
+
31
+ # The number of tokens accepted by the scoring model and verification
32
+ # routine, e.g. Llama2-70B and lossless rejection sampling.
33
+ #
34
+ # NOTE: Any token accepted by the verification routine is considered
35
+ # accepted (regardless of if the speculative prefix is also accepted). The
36
+ # user will usually see less accepted tokens. This metric is helpful when
37
+ # evaluating alignment of the proposal method with the scoring model.
38
+ accepted_tokens: int
39
+
40
+ # The number of speculative tokens per sequence.
41
+ num_spec_tokens: int
42
+
43
+
44
+Timer = Callable[[], float]
45
+
46
+
47
+class AsyncMetricsCollector:
48
+ """Class which copies rejection sampler metrics from the device to CPU on a
49
+ non-default Torch stream.
50
+ """
51
+
52
+ def __init__(self,
53
+ rejection_sampler: RejectionSampler,
54
+ timer: Optional[Timer] = None,
55
+ collect_interval_s: float = 5.0):
56
+ self._rejection_sampler = rejection_sampler
57
+ self._timer = time.time if timer is None else timer
58
+
59
+ self._rank: Optional[int] = None
60
+
61
+ # We don't have a device set yet.
62
+ self._copy_stream: Optional[torch.cuda.Stream] = None
63
+
64
+ self._in_flight_copy: Optional[torch.cuda.Event] = None
65
+
66
+ pin_memory = not in_wsl()
67
+ self._aggregate_num_accepted_tokens = torch.tensor(
68
+ 0, dtype=torch.long, device="cpu", pin_memory=pin_memory)
69
+ self._aggregate_num_emitted_tokens = torch.tensor(
70
+ 0, dtype=torch.long, device="cpu", pin_memory=pin_memory)
71
+ self._aggregate_num_draft_tokens = 0
72
+
73
+ self._rejsample_metrics_collect_interval_s = collect_interval_s
74
+ self._last_metrics_collect_time = self._timer()
75
+
76
+ def init_gpu_tensors(self, rank: int) -> None:
77
+ self._rank = rank
78
+ self._copy_stream = torch.cuda.Stream()
79
+
80
+ def maybe_collect_rejsample_metrics(
81
+ self, k: int) -> Optional[SpecDecodeWorkerMetrics]:
82
+
83
+ # If a copy was initiated in the previous call, collect and return.
84
+ if self._in_flight_copy is not None:
85
+ ready_event = self._in_flight_copy
86
+ self._in_flight_copy = None
87
+ return self._collect_rejsample_metrics(k, ready_event)
88
+
89
+ # Otherwise, check if we should start a new copy.
90
+ if self._should_collect_rejsample_metrics(self._timer()):
91
+ assert self._in_flight_copy is None
92
+ self._in_flight_copy = self._copy_rejsample_metrics_async()
93
+
94
+ return None
95
+
96
+ def _should_collect_rejsample_metrics(self, now: float) -> bool:
97
+ """Return whether or not this iteration should print rejection sampling
98
+ metrics.
99
+ """
100
+ if self._rank != 0:
101
+ return False
102
+
103
+ if (now - self._last_metrics_collect_time <
104
+ self._rejsample_metrics_collect_interval_s):
105
+ return False
106
+ return True
107
+
108
+ def _copy_rejsample_metrics_async(self) -> torch.cuda.Event:
109
+ """Copy rejection sampling metrics (number of accepted tokens, etc) to
110
+ CPU asynchronously.
111
+
112
+ Returns a CUDA event recording when the copy is complete.
113
+ """
114
+ self._copy_stream.wait_stream(torch.cuda.current_stream())
115
+
116
+ with torch.cuda.stream(self._copy_stream):
117
+ self._aggregate_num_accepted_tokens.copy_(
118
+ self._rejection_sampler.num_accepted_tokens, non_blocking=True)
119
+ self._aggregate_num_emitted_tokens.copy_(
120
+ self._rejection_sampler.num_emitted_tokens, non_blocking=True)
121
+ # Number of draft tokens is calculated on CPU, so no copy is
122
+ # required.
123
+ self._aggregate_num_draft_tokens = (
124
+ self._rejection_sampler.num_draft_tokens)
125
+
126
+ aggregate_metrics_ready = torch.cuda.Event()
127
+ aggregate_metrics_ready.record(self._copy_stream)
128
+
129
+ return aggregate_metrics_ready
130
+
131
+ def _collect_rejsample_metrics(
132
+ self, k: int,
133
+ ready_event: torch.cuda.Event) -> SpecDecodeWorkerMetrics:
134
+ """Create metrics object from statistics copied asynchronously.
135
+
136
+ Args:
137
+ k: int. The number of speculative tokens; used to determine system
138
+ efficiency.
139
+ ready_event: torch.cuda.Event. The CUDA event recording when the
140
+ async GPU->CPU copy is complete.
141
+ """
142
+
143
+ ready_event.synchronize()
144
+ accepted_tokens = self._aggregate_num_accepted_tokens.item()
145
+ emitted_tokens = self._aggregate_num_emitted_tokens.item()
146
+ draft_tokens = self._aggregate_num_draft_tokens
147
+
148
+ num_possible_tokens = self.get_max_num_accepted_tokens(draft_tokens, k)
149
+
150
+ if draft_tokens > 0:
151
+ draft_acceptance_rate = accepted_tokens / draft_tokens
152
+ else:
153
+ draft_acceptance_rate = float("nan")
154
+
155
+ if num_possible_tokens > 0:
156
+ system_efficiency = emitted_tokens / num_possible_tokens
157
+ else:
158
+ system_efficiency = float("nan")
159
+
160
+ return SpecDecodeWorkerMetrics(
161
+ num_spec_tokens=k,
162
+ draft_acceptance_rate=draft_acceptance_rate,
163
+ system_efficiency=system_efficiency,
164
+ accepted_tokens=accepted_tokens,
165
+ draft_tokens=draft_tokens,
166
+ emitted_tokens=emitted_tokens,
167
+ )
168
+
169
+ @staticmethod
170
+ def get_max_num_accepted_tokens(draft_tokens: int, k: int) -> int:
171
+ # Divide by k since batch size can be variable.
172
+ total_num_spec_seqs = draft_tokens / k
173
+ num_accepted_per_seq_if_all_accepted = k + 1
174
+ return int(total_num_spec_seqs / num_accepted_per_seq_if_all_accepted)
vllm/spec_decode/multi_step_worker.py
new
+366
@@ -0,0 +1,366 @@
1
+from typing import List, Dict, Optional, Tuple
2
+import copy
3
+
4
+import torch
5
+
6
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
+from vllm.worker.worker import Worker
8
+from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeProposer
9
+from vllm.spec_decode.util import sampler_output_to_torch
10
+
11
+
12
+class MultiStepWorker(Worker):
13
+ """The MultiStepWorker is equivalent to a Worker except that it allows
14
+ multiple forward passes in a single call, assuming the scheduler has
15
+ allocated enough space to store the additional KV. This reduces overhead
16
+ by invoking the scheduler less.
17
+
18
+ The MultiStepWorker does not support cache swap operations, or beam search.
19
+ Cache swap operations do not require large modifications. On the other hand,
20
+ beam search requires memory allocations during sequence forks and thus
21
+ requires more thought for MultiStepWorker support.
22
+ """
23
+
24
+ def __init__(self, *args, **kwargs):
25
+ super().__init__(*args, **kwargs)
26
+
27
+ self._proposer: Optional[DraftModelTop1Proposer] = None
28
+
29
+ def init_model(self):
30
+ super().init_model()
31
+
32
+ self._proposer = DraftModelTop1Proposer(
33
+ self,
34
+ self.device,
35
+ self.max_model_len,
36
+ self.vocab_size,
37
+ )
38
+
39
+ @torch.inference_mode()
40
+ def execute_model_multi_step(
41
+ self,
42
+ seq_group_metadata_list: List[SequenceGroupMetadata],
43
+ blocks_to_swap_in: Dict[int, int],
44
+ blocks_to_swap_out: Dict[int, int],
45
+ blocks_to_copy: Dict[int, List[int]],
46
+ num_steps: int,
47
+ ) -> List[SamplerOutput]:
48
+ """Run the model forward pass num_steps times. Returns the list of
49
+ sampler output, one per model forward pass.
50
+ """
51
+ self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
52
+ blocks_to_swap_out, blocks_to_copy)
53
+
54
+ # Shallow copy input data so modifications (such as appending tokens)
55
+ # do not cause side-effects.
56
+ copied_seq_group_metadata_list = self._shallow_copy_inputs(
57
+ seq_group_metadata_list)
58
+
59
+ # Assert enough KV space for num_steps tokens per sequence.
60
+ self._assert_enough_kv_space(seq_group_metadata_list, num_steps)
61
+
62
+ # Run model num_steps times.
63
+ model_outputs = []
64
+ for _ in range(num_steps):
65
+ model_output = super().execute_model(
66
+ seq_group_metadata_list=copied_seq_group_metadata_list,
67
+ blocks_to_swap_in=blocks_to_swap_in,
68
+ blocks_to_swap_out=blocks_to_swap_out,
69
+ blocks_to_copy=blocks_to_copy,
70
+ )
71
+
72
+ self._append_new_tokens(model_output,
73
+ copied_seq_group_metadata_list)
74
+ model_outputs.append(model_output)
75
+
76
+ return model_outputs
77
+
78
+ def get_spec_proposals(
79
+ self,
80
+ seq_group_metadata_list: List[SequenceGroupMetadata],
81
+ blocks_to_swap_in: Dict[int, int],
82
+ blocks_to_swap_out: Dict[int, int],
83
+ blocks_to_copy: Dict[int, List[int]],
84
+ max_proposal_len: int,
85
+ ) -> SpeculativeProposals:
86
+ """Produce speculations given an input batch of sequences. The number of
87
+ speculative tokens per sequence is determined by max_proposal_len.
88
+ """
89
+
90
+ return self._proposer.get_proposals(
91
+ seq_group_metadata_list,
92
+ blocks_to_swap_in,
93
+ blocks_to_swap_out,
94
+ blocks_to_copy,
95
+ max_proposal_len,
96
+ )
97
+
98
+ def _append_new_tokens(
99
+ self, model_output: SamplerOutput,
100
+ seq_group_metadata_list: SequenceGroupMetadata) -> None:
101
+ """Given model output from a single run, append the tokens to the
102
+ sequences. This is normally done outside of the worker, but it is
103
+ required if the worker is to perform multiple forward passes.
104
+ """
105
+ for seq_group_metadata, sequence_group_outputs in zip(
106
+ seq_group_metadata_list, model_output):
107
+ seq_group_metadata.is_prompt = False
108
+
109
+ for seq_output in sequence_group_outputs.samples:
110
+ # NOTE: Beam search is not supported, so we can assume that
111
+ # parent_seq_id == seq_id.
112
+ seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
113
+
114
+ token_id = seq_output.output_token
115
+ token_logprob = seq_output.logprobs[token_id]
116
+
117
+ seq.append_token_id(token_id, token_logprob.logprob)
118
+
119
+ def _shallow_copy_inputs(
120
+ self, seq_group_metadata_list: List[SequenceGroupMetadata]
121
+ ) -> List[SequenceGroupMetadata]:
122
+ """Copy input data structures to remove side-effects when input data
123
+ structures are shared with other modules.
124
+
125
+ Helpful when the vLLM scheduler runs in the same process as the worker.
126
+ The alternative is deep-copying (or other form of deep copy); this has
127
+ performance downsides.
128
+ """
129
+
130
+ # Shallow-copy the list of SequenceGroupMetadata. This allows us to
131
+ # append tokens and change is_prompt without external side-effects.
132
+ new_seq_group_metadata_list = []
133
+
134
+ for old_seq_group_metadata in seq_group_metadata_list:
135
+ # We must shallow-copy seq_group_metadata as is_prompt could change.
136
+ seq_group_metadata = copy.copy(old_seq_group_metadata)
137
+ new_seq_group_metadata_list.append(seq_group_metadata)
138
+
139
+ # We must shallow-copy seq_data as we will append token ids
140
+ new_seq_data = {}
141
+ for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
142
+ new_seq_data[seq_id] = copy.copy(old_seq_data)
143
+ new_seq_data[
144
+ seq_id].output_token_ids = old_seq_data.output_token_ids[:]
145
+
146
+ seq_group_metadata.seq_data = new_seq_data
147
+
148
+ return new_seq_group_metadata_list
149
+
150
+ def _assert_enough_kv_space(
151
+ self, seq_group_metadata_list: List[SequenceGroupMetadata],
152
+ num_steps: int) -> None:
153
+ """Assert there are enough physical blocks per sequence to store the
154
+ current KV plus additional KV from num_steps tokens.
155
+ """
156
+ assert self.model_runner.block_size is not None
157
+ for seq_group_metadata in seq_group_metadata_list:
158
+ # Only one seq_id is guaranteed because there is no beam search.
159
+ seq_id = list(seq_group_metadata.seq_data.keys())[0]
160
+ seq = seq_group_metadata.seq_data[seq_id]
161
+
162
+ # After num_steps, the seq len will be the current seq len
163
+ # plus one token per step.
164
+ final_seq_len = seq.get_len() + num_steps
165
+
166
+ # We will have final_seq_len - 1 KV because vLLM saves KV for a
167
+ # token in the iteration after the token was generated.
168
+ required_num_kv_slots = final_seq_len - 1
169
+
170
+ # The allocated number of kv slots is the number of allocated blocks
171
+ # times the number of slots of block.
172
+ number_physical_blocks = len(
173
+ seq_group_metadata.block_tables[seq_id])
174
+ allocated_kv_slots = (number_physical_blocks *
175
+ self.model_runner.block_size)
176
+
177
+ if required_num_kv_slots > allocated_kv_slots:
178
+ request_id = seq_group_metadata.request_id
179
+ raise ValueError(
180
+ "The worker attempted to run "
181
+ f"{num_steps} times but found insufficient KV space for "
182
+ f"{request_id=} {seq_id=}. ({allocated_kv_slots=} "
183
+ f"{required_num_kv_slots=}).")
184
+
185
+ def _raise_if_unsupported(
186
+ self,
187
+ seq_group_metadata_list: List[SequenceGroupMetadata],
188
+ blocks_to_swap_in: Dict[int, int],
189
+ blocks_to_swap_out: Dict[int, int],
190
+ blocks_to_copy: Dict[int, List[int]],
191
+ ) -> None:
192
+ """MultiStepWorker does not yet implement support for cache swap
193
+ operations or beam search.
194
+ """
195
+ if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
196
+ raise NotImplementedError(
197
+ "MultiStepWorker does not support cache operations")
198
+
199
+ if any(
200
+ len(seq_group_metadata.seq_data.keys()) != 1
201
+ for seq_group_metadata in seq_group_metadata_list):
202
+ raise NotImplementedError(
203
+ "MultiStepWorker does not support beam search.")
204
+
205
+
206
+class DraftModelTop1Proposer(SpeculativeProposer):
207
+ """Helper class which separates out sequences which would exceed the max
208
+ model length when speculated upon.
209
+
210
+ This allows combinations of models such as JackFram/llama-68m draft with
211
+ meta-llama/Llama2-13b-chat-hf, as llama-68m has max_position_embeddings of
212
+ 2048 while Llama2-13b has max_position_embeddings of 4096.
213
+
214
+ We treat the sequences which exceed the proposal draft model length as
215
+ "non-spec sequences". Essentially they skip the draft model and go through
216
+ normal decoding in the target model.
217
+
218
+ Currently, only proposal_lens of 0 and k are supported, where k is a global
219
+ batch proposal length. In the future vLLM should support per-sequence
220
+ proposal lengths.
221
+ """
222
+
223
+ def __init__(
224
+ self,
225
+ draft_worker: MultiStepWorker,
226
+ device: str,
227
+ max_model_len: int,
228
+ vocab_size: int,
229
+ ):
230
+ self._draft_worker = draft_worker
231
+ self._device = device
232
+ self._max_model_len = max_model_len
233
+ self._vocab_size = vocab_size
234
+
235
+ def get_proposals(
236
+ self,
237
+ seq_group_metadata_list: List[SequenceGroupMetadata],
238
+ blocks_to_swap_in: Dict[int, int],
239
+ blocks_to_swap_out: Dict[int, int],
240
+ blocks_to_copy: Dict[int, List[int]],
241
+ max_proposal_len: int,
242
+ ) -> SpeculativeProposals:
243
+ """Get speculative proposals given the input batch.
244
+
245
+ Sequences which would exceed the max model length are skipped during
246
+ speculation.
247
+ """
248
+
249
+ # Split speculative- and non-speculative- sequences.
250
+ proposal_lens, nonzero_proposal_len_seqs, nonzero_proposal_len_indices = self._split_by_max_model_len(
251
+ seq_group_metadata_list, max_proposal_len)
252
+
253
+ if nonzero_proposal_len_seqs:
254
+ # Speculate tokens using the draft worker for the speculative
255
+ # sequences.
256
+ maybe_sampler_output = self._draft_worker.execute_model_multi_step(
257
+ seq_group_metadata_list=nonzero_proposal_len_seqs,
258
+ blocks_to_swap_in=blocks_to_swap_in,
259
+ blocks_to_swap_out=blocks_to_swap_out,
260
+ blocks_to_copy=blocks_to_copy,
261
+ num_steps=max_proposal_len,
262
+ )
263
+ else:
264
+ # If no sequences can be speculated, set sampler output to None.
265
+ maybe_sampler_output = None
266
+
267
+ # Combine speculative- and non-speculative sequences into the same
268
+ # representation.
269
+ proposal_tokens, proposal_probs, proposal_lens = self._merge_outputs(
270
+ batch_size=len(seq_group_metadata_list),
271
+ max_proposal_len=max_proposal_len,
272
+ maybe_sampler_output=maybe_sampler_output,
273
+ proposal_lens=proposal_lens,
274
+ nonzero_proposal_len_indices=nonzero_proposal_len_indices,
275
+ )
276
+
277
+ proposals = SpeculativeProposals(
278
+ proposal_token_ids=proposal_tokens,
279
+ proposal_probs=proposal_probs,
280
+ proposal_lens=proposal_lens,
281
+ )
282
+
283
+ return proposals
284
+
285
+ def _split_by_max_model_len(
286
+ self,
287
+ seq_group_metadata_list: List[SequenceGroupMetadata],
288
+ max_proposal_len: int,
289
+ ) -> Tuple[List[int], List[SequenceGroupMetadata], List[int]]:
290
+ """Determine which sequences would exceed the max model length.
291
+ """
292
+
293
+ proposal_lens: List[int] = []
294
+ nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
295
+ nonzero_proposal_len_indices: List[int] = []
296
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
297
+ seq_data = next(iter(seq_group_metadata.seq_data.values()))
298
+ seq_len = seq_data.get_len()
299
+
300
+ # Currently only proposal lens of 0 or the global batch proposal len
301
+ # are supported.
302
+ if seq_len + max_proposal_len < self._max_model_len:
303
+ proposal_lens.append(max_proposal_len)
304
+ nonzero_proposal_len_seqs.append(seq_group_metadata)
305
+ nonzero_proposal_len_indices.append(i)
306
+ else:
307
+ proposal_lens.append(0)
308
+
309
+ return proposal_lens, nonzero_proposal_len_seqs, nonzero_proposal_len_indices
310
+
311
+ def _merge_outputs(
312
+ self,
313
+ batch_size: int,
314
+ max_proposal_len: int,
315
+ maybe_sampler_output: Optional[SamplerOutput],
316
+ proposal_lens: List[int],
317
+ nonzero_proposal_len_indices: List[int],
318
+ ) -> Tuple[torch.Tensor, torch.tensor, torch.Tensor]:
319
+ """After speculations are produced, merge the speculation results with
320
+ the skipped sequences.
321
+ """
322
+ if maybe_sampler_output is None:
323
+ # If no speculative tokens, the sampler output will be None.
324
+ # In this case we return empty tensors.
325
+ proposal_tokens = torch.zeros(0,
326
+ max_proposal_len,
327
+ dtype=torch.long,
328
+ device=self._device)
329
+ proposal_probs = torch.zeros(0,
330
+ max_proposal_len,
331
+ self._vocab_size,
332
+ dtype=torch.float32,
333
+ device=self._device)
334
+ proposal_lens = torch.zeros(len(proposal_lens),
335
+ dtype=torch.long,
336
+ device=self._device)
337
+ return proposal_tokens, proposal_probs, proposal_lens
338
+
339
+ sampler_output = maybe_sampler_output
340
+
341
+ proposal_tokens, proposal_probs = sampler_output_to_torch(
342
+ sampler_output)
343
+
344
+ # Now, reformat the output GPU tensors such that each sequence has
345
+ # a proposal. the proposal can be empty, e.g. [-1, -1, -1]
346
+
347
+ entire_proposal_tokens = torch.full(size=(batch_size,
348
+ *proposal_tokens.shape[1:]),
349
+ fill_value=-1,
350
+ dtype=torch.long,
351
+ device=self._device)
352
+ entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens
353
+ entire_proposal_probs = torch.zeros(batch_size,
354
+ *proposal_probs.shape[1:],
355
+ dtype=torch.float32,
356
+ device=self._device)
357
+ entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
358
+
359
+ proposal_tokens, proposal_probs = entire_proposal_tokens, entire_proposal_probs
360
+
361
+ proposal_lens = torch.zeros(batch_size,
362
+ dtype=torch.long,
363
+ device=self._device)
364
+ proposal_lens[nonzero_proposal_len_indices] = max_proposal_len
365
+
366
+ return proposal_tokens, proposal_probs, proposal_lens
vllm/spec_decode/spec_decode_worker.py
new
+372
@@ -0,0 +1,372 @@
1
+from typing import List, Tuple, Optional, Dict
2
+from functools import cached_property
3
+
4
+import torch
5
+
6
+from vllm.spec_decode.metrics import AsyncMetricsCollector
7
+from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
8
+ SequenceGroupOutput, SequenceOutput)
9
+from vllm.worker.worker import Worker
10
+from vllm.spec_decode.multi_step_worker import MultiStepWorker
11
+from vllm.model_executor.layers.rejection_sampler import RejectionSampler
12
+from vllm.config import CacheConfig
13
+from vllm.spec_decode.util import nvtx_range, get_all_seq_ids, split_batch_by_proposal_len
14
+from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeScores
15
+from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
16
+from vllm.spec_decode.interfaces import SpeculativeScorer
17
+
18
+
19
+class SpecDecodeWorker:
20
+ """Worker which implements speculative decoding.
21
+
22
+ Speculative decoding reduces decoding per-token latency by using a proposal
23
+ method, such as a small draft model, to speculate ahead of a larger LLM. The
24
+ probabilities of the speculative tokens are then determined by the larger
25
+ LLM, after which some verification routine determines which (if any) of the
26
+ speculative tokens are accepted by the larger LLM.
27
+
28
+ See https://github.com/vllm-project/vllm/pull/2188 and
29
+ https://github.com/vllm-project/vllm/pull/3103 for more info.
30
+
31
+ The current implementation has the following limitations:
32
+ * Only draft-model proposal is implemented (contributions for more forms are
33
+ welcome!).
34
+ * Only top-1 proposal and scoring are implemented. Tree-attention is left as
35
+ future work.
36
+ * Only lossless rejection sampling is supported. Contributions adding lossy
37
+ verification routines are welcome (e.g. Medusa's typical acceptance).
38
+ * All sequences in a batch must have the same proposal length, or zero. This
39
+ can be improved by having per-sequence speculation in the future.
40
+ * The scoring forward pass is done without an MQA kernel, which is
41
+ suboptimal especially as the batch size, proposal length, and sequence
42
+ lengths grow. Contributions to add a MQA scoring are welcome once
43
+ correctness tests pass.
44
+ More info here https://docs.google.com/document/d/1T-JaS2T1NRfdP51qzqpyakoCXxSXTtORppiwaj5asxA/edit.
45
+ """
46
+
47
+ def __init__(
48
+ self,
49
+ proposer_worker: MultiStepWorker,
50
+ scorer_worker: Worker,
51
+ rejection_sampler: RejectionSampler,
52
+ metrics_collector: Optional[AsyncMetricsCollector] = None,
53
+ ):
54
+ """
55
+ Create a SpecDecodeWorker.
56
+
57
+ Args:
58
+ proposer_worker: A worker that can produce speculative tokens for
59
+ sequences.
60
+ scorer_worker: A worker that produces probabilities of speculative
61
+ tokens according to some base model. Typically a vanilla vLLM
62
+ Worker.
63
+ rejection_sampler: A Torch module used to perform modified rejection
64
+ sampling for speculative decoding.
65
+ metrics_collector: Helper class for collecting metrics; can be set
66
+ for testing purposes.
67
+ """
68
+ self.proposer_worker = proposer_worker
69
+ self.scorer_worker = scorer_worker
70
+ self.rejection_sampler = rejection_sampler
71
+
72
+ self._metrics = AsyncMetricsCollector(
73
+ rejection_sampler
74
+ ) if metrics_collector is None else metrics_collector
75
+
76
+ self.probs_dtype = self.rejection_sampler.probs_dtype
77
+ self.token_id_dtype = self.rejection_sampler.token_id_dtype
78
+
79
+ self.scorer: SpeculativeScorer = None
80
+
81
+ def init_model(self) -> None:
82
+ """Initialize both scorer and proposer models.
83
+ """
84
+ # The scorer worker model is initialized first in case the proposer
85
+ # model has a smaller TP degree than the target worker.
86
+ self.scorer_worker.init_model()
87
+ self.proposer_worker.init_model()
88
+
89
+ self._metrics.init_gpu_tensors(self.rank)
90
+ self.rejection_sampler.init_gpu_tensors(self.rank)
91
+ self.scorer = BatchExpansionTop1Scorer(
92
+ scorer_worker=self.scorer_worker,
93
+ device=self.device,
94
+ vocab_size=self._vocab_size)
95
+
96
+ def profile_num_available_blocks(self, block_size: int,
97
+ gpu_memory_utilization: float,
98
+ cpu_swap_space: int,
99
+ cache_dtype: str) -> Tuple[int, int]:
100
+ """Determine the number of cache blocks to use.
101
+
102
+ This is done by profiling the scorer model (which is typically the
103
+ larger of the two). Then the total memory which would be used by the
104
+ scorer cache is divided evenly between the proposer and scorer model KV,
105
+ such that the number of blocks is equal in both KV caches.
106
+ """
107
+ num_gpu_blocks, num_cpu_blocks = (
108
+ self.scorer_worker.profile_num_available_blocks(
109
+ block_size, gpu_memory_utilization, cpu_swap_space,
110
+ cache_dtype))
111
+
112
+ scorer_cache_block_size_bytes = self.scorer_worker.get_cache_block_size_bytes(
113
+ block_size, cache_dtype)
114
+ proposer_cache_block_size_bytes = self.proposer_worker.get_cache_block_size_bytes(
115
+ block_size, cache_dtype)
116
+
117
+ new_num_gpu_blocks = split_num_cache_blocks_evenly(
118
+ scorer_cache_block_size_bytes, proposer_cache_block_size_bytes,
119
+ num_gpu_blocks)
120
+ return new_num_gpu_blocks, num_cpu_blocks
121
+
122
+ def init_cache_engine(self, cache_config: CacheConfig):
123
+ """Initialize the cache engine of the scorer and proposer workers.
124
+ """
125
+ self.scorer_worker.init_cache_engine(cache_config)
126
+ self.proposer_worker.init_cache_engine(cache_config)
127
+
128
+ @torch.inference_mode()
129
+ def execute_model(
130
+ self,
131
+ seq_group_metadata_list: List[SequenceGroupMetadata],
132
+ blocks_to_swap_in: Optional[Dict[int, int]],
133
+ blocks_to_swap_out: Optional[Dict[int, int]],
134
+ blocks_to_copy: Optional[Dict[int, List[int]]],
135
+ num_spec_tokens: int,
136
+ ) -> List[SamplerOutput]:
137
+ """Perform speculative decoding on the input batch.
138
+ """
139
+
140
+ assert seq_group_metadata_list is not None, (
141
+ "speculative decoding "
142
+ "requires non-None seq_group_metadata_list")
143
+
144
+ # If no spec tokens, call the proposer and scorer workers normally.
145
+ # Used for prefill.
146
+ if num_spec_tokens == 0 or len(seq_group_metadata_list) == 0:
147
+ return self._run_no_spec(
148
+ seq_group_metadata_list=seq_group_metadata_list,
149
+ blocks_to_swap_in=blocks_to_swap_in,
150
+ blocks_to_swap_out=blocks_to_swap_out,
151
+ blocks_to_copy=blocks_to_copy,
152
+ )
153
+
154
+ return self._run_speculative_decoding_step(
155
+ seq_group_metadata_list=seq_group_metadata_list,
156
+ blocks_to_swap_in=blocks_to_swap_in,
157
+ blocks_to_swap_out=blocks_to_swap_out,
158
+ blocks_to_copy=blocks_to_copy,
159
+ k=num_spec_tokens,
160
+ )
161
+
162
+ @nvtx_range("spec_decode_worker._run_no_spec")
163
+ def _run_no_spec(
164
+ self,
165
+ seq_group_metadata_list: List[SequenceGroupMetadata],
166
+ blocks_to_swap_in: Optional[Dict[int, int]],
167
+ blocks_to_swap_out: Optional[Dict[int, int]],
168
+ blocks_to_copy: Optional[Dict[int, List[int]]],
169
+ ) -> List[SamplerOutput]:
170
+ """Run a prefill step, without any speculation. The input is sent to the
171
+ proposer and scorer model so that the KV cache is consistent between the
172
+ two.
173
+ """
174
+
175
+ self.proposer_worker.execute_model(
176
+ seq_group_metadata_list=seq_group_metadata_list,
177
+ blocks_to_swap_in=blocks_to_swap_in,
178
+ blocks_to_swap_out=blocks_to_swap_out,
179
+ blocks_to_copy=blocks_to_copy,
180
+ return_python_output=False)
181
+
182
+ sampler_output = self.scorer_worker.execute_model(
183
+ seq_group_metadata_list=seq_group_metadata_list,
184
+ blocks_to_swap_in=blocks_to_swap_in,
185
+ blocks_to_swap_out=blocks_to_swap_out,
186
+ blocks_to_copy=blocks_to_copy,
187
+ )
188
+
189
+ # Clear device tensors from sampler output. This reduces communication
190
+ # overhead when the engine runs in a different process than the workers.
191
+ sampler_output.probs = None
192
+ sampler_output.sampled_tokens = None
193
+ return [sampler_output]
194
+
195
+ @nvtx_range("spec_decode_worker._run_speculative_decoding_step")
196
+ def _run_speculative_decoding_step(
197
+ self,
198
+ seq_group_metadata_list: List[SequenceGroupMetadata],
199
+ blocks_to_swap_in: Optional[Dict[int, int]],
200
+ blocks_to_swap_out: Optional[Dict[int, int]],
201
+ blocks_to_copy: Optional[Dict[int, List[int]]],
202
+ k: int,
203
+ ) -> List[SamplerOutput]:
204
+ """Execute a single step of speculative decoding.
205
+
206
+ This invokes the proposer worker to get k speculative tokens for each
207
+ sequence, then scores each speculative token using the scoring worker.
208
+
209
+ Returns a list of SamplerOutput, each containing a single token per
210
+ sequence.
211
+ """
212
+
213
+ # Generate proposals using draft worker.
214
+ proposals = self.proposer_worker.get_spec_proposals(
215
+ seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
216
+ blocks_to_copy, k)
217
+
218
+ proposal_scores = self.scorer.score_proposals(
219
+ seq_group_metadata_list,
220
+ blocks_to_swap_in,
221
+ blocks_to_swap_out,
222
+ blocks_to_copy,
223
+ k,
224
+ proposals,
225
+ )
226
+
227
+ accepted_token_ids = self._verify_tokens(seq_group_metadata_list,
228
+ proposal_scores, proposals, k)
229
+
230
+ return self._create_output_sampler_list(seq_group_metadata_list,
231
+ accepted_token_ids, k)
232
+
233
+ @nvtx_range("spec_decode_worker._verify_tokens")
234
+ def _verify_tokens(
235
+ self,
236
+ seq_group_metadata_list: List[SequenceGroupMetadata],
237
+ proposal_scores: SpeculativeScores,
238
+ proposals: SpeculativeProposals,
239
+ max_proposal_len: int,
240
+ ) -> torch.Tensor:
241
+ """Determine which speculative tokens are accepted using the
242
+ probabilities of each token according to the proposer and scorer models.
243
+ """
244
+ proposal_lens_list = proposals.proposal_lens.tolist()
245
+
246
+ # vLLM currently only supports proposal lens equal to zero or the batch
247
+ # proposal len. This adds some complexity (splitting the batch into spec
248
+ # and non spec sequences) and should be removed in the future. It can be
249
+ # done by supporting per-sequence proposal lens.
250
+ _, spec_indices = split_batch_by_proposal_len(
251
+ seq_group_metadata_list,
252
+ proposal_lens_list,
253
+ select_proposal_len_zero=False)
254
+ _, non_spec_indices = split_batch_by_proposal_len(
255
+ seq_group_metadata_list,
256
+ proposal_lens_list,
257
+ select_proposal_len_zero=True)
258
+ original_indices = spec_indices + non_spec_indices
259
+
260
+ proposal_probs = proposal_scores.probs[spec_indices, :-1]
261
+ bonus_token_ids = proposal_scores.token_ids[spec_indices, -1:]
262
+ non_spec_token_ids = proposal_scores.token_ids[non_spec_indices]
263
+
264
+ accepted_token_ids = self.rejection_sampler(
265
+ proposal_probs,
266
+ bonus_token_ids,
267
+ proposals.proposal_probs,
268
+ proposals.proposal_token_ids,
269
+ )
270
+
271
+ # Append output tokens from non-speculative sequences to
272
+ # the accepted token ids tensor.
273
+ non_spec_token_ids = non_spec_token_ids.expand(-1, max_proposal_len +
274
+ 1).clone()
275
+ non_spec_token_ids[:, 1:] = -1
276
+ accepted_token_ids = torch.cat(
277
+ [accepted_token_ids, non_spec_token_ids])
278
+
279
+ # Rearrange so that results are in the order of the original seq group
280
+ # metadata.
281
+ accepted_token_ids[original_indices] = accepted_token_ids.clone()
282
+
283
+ return accepted_token_ids
284
+
285
+ def _create_output_sampler_list(
286
+ self,
287
+ seq_group_metadata_list: List[SequenceGroupMetadata],
288
+ accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1]
289
+ k: int,
290
+ ) -> List[SamplerOutput]:
291
+ """Given the accepted token ids, create a list of SamplerOutput.
292
+
293
+ The output is padded with -1 tokens such that each sequence has
294
+ the same number of outputs.
295
+ """
296
+ seq_ids = get_all_seq_ids(seq_group_metadata_list)
297
+
298
+ # shape: [k+1, batch_size]
299
+ accepted_token_ids_by_step = accepted_token_ids.transpose(0,
300
+ 1).tolist()
301
+ sampler_output_list = []
302
+ for token_ids_by_step in accepted_token_ids_by_step:
303
+ if all(token_id == -1 for token_id in token_ids_by_step):
304
+ break
305
+
306
+ step_output_token_ids = []
307
+ for token_id, seq_id in zip(token_ids_by_step, seq_ids):
308
+ step_output_token_ids.append(
309
+ SequenceGroupOutput(
310
+ samples=[
311
+ SequenceOutput(
312
+ parent_seq_id=seq_id,
313
+ output_token=token_id,
314
+ # TODO Add verifier logprobs.
315
+ logprobs={token_id: 0.0},
316
+ )
317
+ ],
318
+ prompt_logprobs=None,
319
+ ))
320
+ sampler_output_list.append(
321
+ SamplerOutput(outputs=step_output_token_ids))
322
+
323
+ maybe_rejsample_metrics = self._metrics.maybe_collect_rejsample_metrics(
324
+ k)
325
+ if maybe_rejsample_metrics is not None:
326
+ sampler_output_list[
327
+ 0].spec_decode_worker_metrics = maybe_rejsample_metrics
328
+
329
+ return sampler_output_list
330
+
331
+ @cached_property
332
+ def _vocab_size(self) -> int:
333
+ """Get the vocab size of the model and make sure it's consistent between
334
+ draft and target workers.
335
+ """
336
+ vocab_sizes = [
337
+ worker.vocab_size
338
+ for worker in [self.proposer_worker, self.scorer_worker]
339
+ ]
340
+ assert all(vocab_sizes[0] == vocab_size for vocab_size in vocab_sizes)
341
+ return vocab_sizes[0]
342
+
343
+ @property
344
+ def rank(self):
345
+ return self.scorer_worker.rank
346
+
347
+ @property
348
+ def device(self):
349
+ return self.scorer_worker.device
350
+
351
+
352
+def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
353
+ proposer_cache_block_size_bytes: int,
354
+ total_num_gpu_blocks: int) -> int:
355
+ """Given total_num_gpu_blocks, the number of GPU blocks that could be
356
+ allocate to the target model, this function calculates how many blocks
357
+ should be given to the draft and target model.
358
+
359
+ Note that usually the block size, in bytes, of each model is different,
360
+ as it's a function of number of KV/layer, number of heads, and hidden
361
+ dimension size.
362
+
363
+ Since the target and draft models allocate the same number of blocks, we
364
+ simply calculate the number of blocks where if allocated by both models,
365
+ the total memory usage from KV cache is no larger than the number of
366
+ blocks allocatable by the target model alone.
367
+ """
368
+ new_num_gpu_blocks = int(
369
+ total_num_gpu_blocks * scorer_cache_block_size_bytes /
370
+ (proposer_cache_block_size_bytes + scorer_cache_block_size_bytes))
371
+
372
+ return new_num_gpu_blocks
vllm/spec_decode/util.py
new
+99
@@ -0,0 +1,99 @@
1
+import torch
2
+from typing import List, Tuple
3
+from vllm.sequence import SequenceGroupMetadata, SamplerOutput
4
+from contextlib import contextmanager
5
+from itertools import chain
6
+
7
+SeqId = int
8
+
9
+
10
+def get_all_seq_ids(
11
+ seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[SeqId]:
12
+ """Given a list of SequenceGroupMetadata, create a list of all
13
+ sequence ids.
14
+ """
15
+ return list(
16
+ chain.from_iterable([
17
+ seq_group_metadata.seq_data.keys()
18
+ for seq_group_metadata in seq_group_metadata_list
19
+ ]))
20
+
21
+
22
+def split_batch_by_proposal_len(
23
+ seq_group_metadata_list: List[SequenceGroupMetadata],
24
+ proposal_lens: List[int], select_proposal_len_zero: bool
25
+) -> Tuple[List[SequenceGroupMetadata], List[int]]:
26
+ """Utility function that splits a batch based on whether the proposal len is
27
+ zero or not. We should remove this once vLLM supports per-sequence proposal
28
+ lens in a batch.
29
+ """
30
+
31
+ if select_proposal_len_zero:
32
+ predicate = lambda proposal_len: proposal_len == 0
33
+ else:
34
+ predicate = lambda proposal_len: proposal_len != 0
35
+
36
+ indices = [
37
+ i for i, (_, proposal_len
38
+ ) in enumerate(zip(seq_group_metadata_list, proposal_lens))
39
+ if predicate(proposal_len)
40
+ ]
41
+ seq_groups = [
42
+ seq_group for seq_group, proposal_len in zip(
43
+ seq_group_metadata_list, proposal_lens) if predicate(proposal_len)
44
+ ]
45
+
46
+ return seq_groups, indices
47
+
48
+
49
+def sampler_output_to_torch(
50
+ sampler_output_list: List[SamplerOutput],
51
+) -> Tuple[torch.Tensor, torch.Tensor]:
52
+ """Utility function which converts a list of SamplerOutput to tensors.
53
+
54
+ Returns:
55
+ sampled_token_ids: torch.Tensor
56
+ shape: [batch_size, len(sampler_output_list)]
57
+
58
+ sampled_token_probs: torch.Tensor
59
+ shape: [batch_size, len(sampler_output_list), vocab_size]
60
+ """
61
+
62
+ # shape: [batch_size, num_sampler_output, vocab_size]
63
+ sampled_token_probs = torch.stack(
64
+ [
65
+ sampler_output.sampled_token_probs
66
+ for sampler_output in sampler_output_list
67
+ ],
68
+ dim=0,
69
+ ).transpose(0, 1)
70
+
71
+ # shape: [batch_size, num_sampler_output]
72
+ sampled_token_ids = torch.stack(
73
+ [
74
+ sampler_output.sampled_token_ids.flatten()
75
+ for sampler_output in sampler_output_list
76
+ ],
77
+ dim=0,
78
+ ).transpose(0, 1)
79
+
80
+ return sampled_token_ids, sampled_token_probs
81
+
82
+
83
+@contextmanager
84
+def nvtx_range(msg, *args, **kwargs):
85
+ """
86
+ Context manager / decorator that pushes an NVTX range at the beginning
87
+ of its scope, and pops it at the end. If extra arguments are given,
88
+ they are passed as arguments to msg.format().
89
+
90
+ If running with cuda graphs, you must enable nsys cuda graph profiling.
91
+
92
+ Arguments:
93
+ msg (string): message to associate with the range
94
+ """
95
+ torch.cuda.nvtx.range_push(msg.format(*args, **kwargs))
96
+ try:
97
+ yield
98
+ finally:
99
+ torch.cuda.nvtx.range_pop()
vllm/worker/model_runner.py
+6
-5
@@ -97,8 +97,6 @@ class ModelRunner:
97
f"Loading model weights took {self.model_memory_usage / float(2**30):.4f} GB"
98
)
99
100
- vocab_size = self.model.config.vocab_size
101
-
100
if self.lora_config:
101
assert hasattr(
102
self.model, "supported_lora_modules"
@@ -111,7 +109,7 @@ class ModelRunner:
109
self.lora_manager = LRUCacheWorkerLoRAManager(
110
self.scheduler_config.max_num_seqs,
111
self.scheduler_config.max_num_batched_tokens +
114
- self.scheduler_config.max_paddings, vocab_size,
112
+ self.scheduler_config.max_paddings, self.vocab_size,
113
self.lora_config, self.device, self.model.embedding_modules,
114
self.model.embedding_padding_modules)
115
self.model = self.lora_manager.create_lora_manager(self.model)
@@ -607,8 +605,7 @@ class ModelRunner:
605
@torch.inference_mode()
606
def profile_run(self) -> None:
607
# Enable top-k sampling to reflect the accurate memory usage.
610
- vocab_size = self.model_config.get_vocab_size()
611
- sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
608
+ sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
609
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
610
max_num_seqs = self.scheduler_config.max_num_seqs
611
@@ -774,6 +771,10 @@ class ModelRunner:
771
self.graph_runners.clear()
772
self.cupy_nccl_backend = None
773
774
+ @property
775
+ def vocab_size(self) -> int:
776
+ return self.model_config.get_vocab_size()
777
+
778
779
class CUDAGraphRunner:
780
vllm/worker/spec_decode/multi_step_worker.py
deleted
-178
@@ -1,178 +0,0 @@
1
-from typing import List, Dict
2
-import copy
3
-
4
-import torch
5
-
6
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
-from vllm.worker.worker import Worker
8
-
9
-
10
-class MultiStepWorker(Worker):
11
- """The MultiStepWorker is equivalent to a Worker except that it allows
12
- multiple forward passes in a single call, assuming the scheduler has
13
- allocated enough space to store the additional KV. This reduces overhead
14
- by invoking the scheduler less.
15
-
16
- The MultiStepWorker does not support cache swap operations, or beam search.
17
- Cache swap operations do not require large modifications. On the other hand,
18
- beam search requires memory allocations during sequence forks and thus
19
- requires more thought for MultiStepWorker support.
20
- """
21
-
22
- @torch.inference_mode()
23
- def execute_model_multi_step(
24
- self,
25
- seq_group_metadata_list: List[SequenceGroupMetadata],
26
- blocks_to_swap_in: Dict[int, int],
27
- blocks_to_swap_out: Dict[int, int],
28
- blocks_to_copy: Dict[int, List[int]],
29
- num_steps: int,
30
- ) -> List[SamplerOutput]:
31
- """Run the model forward pass num_steps times. Returns the list of
32
- sampler output, one per model forward pass.
33
- """
34
- self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
35
- blocks_to_swap_out, blocks_to_copy)
36
-
37
- # Shallow copy input data so modifications (such as appending tokens)
38
- # do not cause side-effects.
39
- copied_seq_group_metadata_list = self._shallow_copy_inputs(
40
- seq_group_metadata_list)
41
-
42
- # Assert enough KV space for num_steps tokens per sequence.
43
- self._assert_enough_kv_space(seq_group_metadata_list, num_steps)
44
-
45
- # Run model num_steps times.
46
- model_outputs = []
47
- for _ in range(num_steps):
48
- model_output = super().execute_model(
49
- seq_group_metadata_list=copied_seq_group_metadata_list,
50
- blocks_to_swap_in=blocks_to_swap_in,
51
- blocks_to_swap_out=blocks_to_swap_out,
52
- blocks_to_copy=blocks_to_copy,
53
- )
54
-
55
- self._append_new_tokens(model_output,
56
- copied_seq_group_metadata_list)
57
- model_outputs.append(model_output)
58
-
59
- return model_outputs
60
-
61
- def _append_new_tokens(
62
- self, model_output: SamplerOutput,
63
- seq_group_metadata_list: SequenceGroupMetadata) -> None:
64
- """Given model output from a single run, append the tokens to the
65
- sequences. This is normally done outside of the worker, but it is
66
- required if the worker is to perform multiple forward passes.
67
- """
68
- for seq_group_metadata, sequence_group_outputs in zip(
69
- seq_group_metadata_list, model_output):
70
- seq_group_metadata.is_prompt = False
71
-
72
- for seq_output in sequence_group_outputs.samples:
73
- # NOTE: Beam search is not supported, so we can assume that
74
- # parent_seq_id == seq_id.
75
- seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
76
-
77
- token_id = seq_output.output_token
78
- token_logprob = seq_output.logprobs[token_id]
79
-
80
- seq.append_token_id(token_id, token_logprob.logprob)
81
-
82
- def _shallow_copy_inputs(
83
- self, seq_group_metadata_list: List[SequenceGroupMetadata]
84
- ) -> List[SequenceGroupMetadata]:
85
- """Copy input data structures to remove side-effects when input data
86
- structures are shared with other modules.
87
-
88
- The multi-step worker must be able to append tokens to sequences after
89
- a forward pass. This necessitates modification of the data structures
90
- used by the worker. Since these data structures are shared with other
91
- parts of vLLM, like the scheduler, we must take care not to introduce
92
- unexpected side-effects.
93
-
94
- When Ray is used to orchestrate worker processes (such as when the
95
- tensor-parallel degree is >1), this is not a problem because the input
96
- datastructures will be serialized and created anew in the worker
97
- process.
98
-
99
- However, when Ray is not used to orchestrate the worker processes (such
100
- as when the tensor-parallel degree is 1), this is a problem. We avoid
101
- the problem by shallow-copying the input datastructures (specifically,
102
- the parts that will change in multiple steps).
103
- """
104
-
105
- # Shallow-copy the list of SequenceGroupMetadata. This allows us to
106
- # append tokens and change is_prompt without external side-effects.
107
- new_seq_group_metadata_list = []
108
-
109
- for old_seq_group_metadata in seq_group_metadata_list:
110
- # We must shallow-copy seq_group_metadata as is_prompt could change.
111
- seq_group_metadata = copy.copy(old_seq_group_metadata)
112
- new_seq_group_metadata_list.append(seq_group_metadata)
113
-
114
- # We must shallow-copy seq_data as we will append token ids
115
- new_seq_data = {}
116
- for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
117
- new_seq_data[seq_id] = copy.copy(old_seq_data)
118
- new_seq_data[
119
- seq_id].output_token_ids = old_seq_data.output_token_ids[:]
120
-
121
- seq_group_metadata.seq_data = new_seq_data
122
-
123
- return new_seq_group_metadata_list
124
-
125
- def _assert_enough_kv_space(
126
- self, seq_group_metadata_list: List[SequenceGroupMetadata],
127
- num_steps: int) -> None:
128
- """Assert there are enough physical blocks per sequence to store the
129
- current KV plus additional KV from num_steps tokens.
130
- """
131
- assert self.model_runner.block_size is not None
132
- for seq_group_metadata in seq_group_metadata_list:
133
- # Only one seq_id is guaranteed because there is no beam search.
134
- seq_id = list(seq_group_metadata.seq_data.keys())[0]
135
- seq = seq_group_metadata.seq_data[seq_id]
136
-
137
- # After num_steps, the seq len will be the current seq len
138
- # plus one token per step.
139
- final_seq_len = seq.get_len() + num_steps
140
-
141
- # We will have final_seq_len - 1 KV because vLLM saves KV for a
142
- # token in the iteration after the token was generated.
143
- required_num_kv_slots = final_seq_len - 1
144
-
145
- # The allocated number of kv slots is the number of allocated blocks
146
- # times the number of slots of block.
147
- number_physical_blocks = len(
148
- seq_group_metadata.block_tables[seq_id])
149
- allocated_kv_slots = (number_physical_blocks *
150
- self.model_runner.block_size)
151
-
152
- if required_num_kv_slots > allocated_kv_slots:
153
- request_id = seq_group_metadata.request_id
154
- raise ValueError(
155
- "The worker attempted to run "
156
- f"{num_steps} times but found insufficient KV space for "
157
- f"{request_id=} {seq_id=}. ({allocated_kv_slots=} "
158
- f"{required_num_kv_slots=}).")
159
-
160
- def _raise_if_unsupported(
161
- self,
162
- seq_group_metadata_list: List[SequenceGroupMetadata],
163
- blocks_to_swap_in: Dict[int, int],
164
- blocks_to_swap_out: Dict[int, int],
165
- blocks_to_copy: Dict[int, List[int]],
166
- ) -> None:
167
- """MultiStepWorker does not yet implement support for cache swap
168
- operations or beam search.
169
- """
170
- if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
171
- raise NotImplementedError(
172
- "MultiStepWorker does not support cache operations")
173
-
174
- if any(
175
- len(seq_group_metadata.seq_data.keys()) != 1
176
- for seq_group_metadata in seq_group_metadata_list):
177
- raise NotImplementedError(
178
- "MultiStepWorker does not support beam search.")
vllm/worker/worker.py
+18
-2
@@ -130,8 +130,8 @@ class Worker:
130
# GPU did not change their memory usage during the profiling.
131
peak_memory = self.init_gpu_memory - free_gpu_memory
132
133
- cache_block_size = CacheEngine.get_cache_block_size(
134
- block_size, cache_dtype, self.model_config, self.parallel_config)
133
+ cache_block_size = self.get_cache_block_size_bytes(
134
+ block_size, cache_dtype)
135
num_gpu_blocks = int(
136
(total_gpu_memory * gpu_memory_utilization - peak_memory) //
137
cache_block_size)
@@ -232,6 +232,22 @@ class Worker:
232
def list_loras(self) -> Set[int]:
233
return self.model_runner.list_loras()
234
235
+ @property
236
+ def max_model_len(self) -> int:
237
+ return self.model_config.max_model_len
238
+
239
+ @property
240
+ def vocab_size(self) -> int:
241
+ return self.model_runner.vocab_size
242
+
243
+ def get_cache_block_size_bytes(self, block_size: int,
244
+ cache_dtype: str) -> int:
245
+ """Get the size of the KV cache block size in bytes.
246
+ """
247
+ return CacheEngine.get_cache_block_size(block_size, cache_dtype,
248
+ self.model_config,
249
+ self.parallel_config)
250
+
251
252
def init_distributed_environment(
253
parallel_config: ParallelConfig,