@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8437bae6ef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Speculative decoding 3/9] Worker which speculates, scores, and applies rejection sampling (#3103)

Cade Daniel committed Mar 8, 2024 at 23:32 UTC 8437bae6ef47a690d18c72f0da02c7e5abe83866
21 files changed +2786 -215
.buildkite/test-pipeline.yaml
+4 -1
@@ -28,7 +28,7 @@ steps:
28 num_gpus: 2 # only support 1 or 2 for now.
29
30 - label: Engine Test
31 - command: pytest -v -s engine
31 + command: pytest -v -s engine test_sequence.py
32
33 - label: Entrypoints Test
34 command: pytest -v -s entrypoints
@@ -52,6 +52,9 @@ steps:
52 - label: Worker Test
53 command: pytest -v -s worker
54
55 +- label: Speculative decoding tests
56 + command: pytest -v -s spec_decode
57 +
58 - label: LoRA Test
59 command: pytest -v -s lora --forked
60
tests/spec_decode/__init__.py renamed
tests/spec_decode/test_batch_expansion.py new
+95
@@ -0,0 +1,95 @@
1 +import torch
2 +import pytest
3 +
4 +from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
5 +
6 +from .utils import mock_worker, create_seq_group_metadata_from_prompts
7 +
8 +
9 +@pytest.mark.parametrize('num_target_seq_ids', [100])
10 +def test_create_target_seq_id_iterator(num_target_seq_ids: int):
11 + """Verify all new sequence ids are greater than all input
12 + seq ids.
13 + """
14 + scorer = BatchExpansionTop1Scorer(mock_worker(), 'cuda:0', 32_000)
15 +
16 + all_seq_ids = [
17 + [1, 3, 5, 7],
18 + list(range(100)) + [0],
19 + [100],
20 + ]
21 +
22 + for seq_ids in all_seq_ids:
23 + max_seq_id = max(seq_ids)
24 + iterator = scorer._create_target_seq_id_iterator(seq_ids) # pylint: disable=protected-access
25 + for _ in range(num_target_seq_ids):
26 + assert next(iterator) > max_seq_id
27 +
28 +
29 +@pytest.mark.parametrize('k', [1, 2, 6])
30 +def test_get_token_ids_to_score(k: int):
31 + """Verify correct tokens are selected for scoring.
32 + """
33 + proposal_token_ids = torch.tensor(
34 + list(range(k)),
35 + dtype=torch.int64,
36 + device='cuda',
37 + )
38 +
39 + expected_output = [
40 + [],
41 + ]
42 + for i in range(proposal_token_ids.shape[0]):
43 + expected_output.append(proposal_token_ids[:i + 1].tolist())
44 +
45 + scorer = BatchExpansionTop1Scorer(mock_worker(), 'cuda:0', 32_000)
46 + actual_output = scorer._get_token_ids_to_score(proposal_token_ids) # pylint: disable=protected-access
47 +
48 + actual_output = [
49 + x.tolist() if isinstance(x, torch.Tensor) else x for x in actual_output
50 + ]
51 +
52 + assert actual_output == expected_output
53 +
54 +
55 +@pytest.mark.parametrize('k', [1, 2, 6])
56 +def test_create_single_target_seq_group_metadata(k: int):
57 + """Verify correct creation of a batch-expanded seq group metadata.
58 + """
59 +
60 + prompt_tokens = [1, 2, 3]
61 + prev_output_tokens = [4, 5, 6]
62 +
63 + token_ids = list(range(k))
64 +
65 + num_tokens_processed = len(prompt_tokens) + len(prev_output_tokens) - 1
66 +
67 + final_seq_len = len(prompt_tokens) + len(prev_output_tokens) + len(
68 + token_ids)
69 +
70 + block_size = 32
71 + input_seq_group_metadata = create_seq_group_metadata_from_prompts(
72 + [prompt_tokens], 2048 // block_size, block_size, [final_seq_len],
73 + [prev_output_tokens], [num_tokens_processed])[0]
74 +
75 + input_seq_id = list(input_seq_group_metadata.seq_data.keys())[0]
76 + target_seq_id = 100
77 +
78 + scorer = BatchExpansionTop1Scorer(mock_worker(), 'cuda:0', 32_000)
79 + output = scorer._create_single_target_seq_group_metadata( # pylint: disable=protected-access
80 + input_seq_group_metadata,
81 + input_seq_id,
82 + target_seq_id,
83 + token_ids,
84 + )
85 +
86 + assert output.request_id == input_seq_group_metadata.request_id
87 + assert len(output.seq_data) == 1
88 + assert output.seq_data[target_seq_id].get_prompt_token_ids(
89 + ) == prompt_tokens
90 + assert output.seq_data[target_seq_id].get_output_token_ids(
91 + ) == prev_output_tokens + token_ids
92 +
93 + assert len(output.block_tables) == 1
94 + assert output.block_tables[
95 + target_seq_id] == input_seq_group_metadata.block_tables[input_seq_id]
tests/spec_decode/test_metrics.py new
+157
@@ -0,0 +1,157 @@
1 +import torch
2 +import math
3 +import pytest
4 +
5 +from unittest.mock import MagicMock
6 +
7 +from vllm.spec_decode.metrics import AsyncMetricsCollector
8 +
9 +
10 +def test_initial_call_returns_none():
11 + """Expect first call to get metrics to return None.
12 + """
13 + rej_sampler = MagicMock()
14 + rej_sampler.num_accepted_tokens = torch.tensor(0,
15 + dtype=torch.long,
16 + device='cuda')
17 + rej_sampler.num_emitted_tokens = torch.tensor(0,
18 + dtype=torch.long,
19 + device='cuda')
20 + rej_sampler.num_draft_tokens = 0
21 +
22 + collector = AsyncMetricsCollector(rej_sampler)
23 + collector.init_gpu_tensors(rank=0)
24 + maybe_metrics = collector.maybe_collect_rejsample_metrics(k=5)
25 + assert maybe_metrics is None
26 +
27 +
28 +def test_second_call_returns_metrics():
29 + """Expect second call to not return None.
30 + """
31 + rej_sampler = MagicMock()
32 + rej_sampler.num_accepted_tokens = torch.tensor(0,
33 + dtype=torch.long,
34 + device='cuda')
35 + rej_sampler.num_emitted_tokens = torch.tensor(0,
36 + dtype=torch.long,
37 + device='cuda')
38 + rej_sampler.num_draft_tokens = 0
39 +
40 + collect_interval_s = 5.0
41 + timer = MagicMock()
42 + timer.side_effect = [
43 + 0.0, collect_interval_s + 0.1, collect_interval_s + 0.2
44 + ]
45 +
46 + collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
47 + timer=timer,
48 + collect_interval_s=collect_interval_s)
49 + collector.init_gpu_tensors(rank=0)
50 + _ = collector.maybe_collect_rejsample_metrics(k=5)
51 + metrics = collector.maybe_collect_rejsample_metrics(k=5)
52 + assert metrics is not None
53 +
54 +
55 +@pytest.mark.parametrize("rank", [1, 2, 3, 4])
56 +def test_nonzero_rank_noop(rank):
57 + """Verify nonzero ranks don't collect metrics.
58 + """
59 + rej_sampler = MagicMock()
60 + rej_sampler.num_accepted_tokens = torch.tensor(0,
61 + dtype=torch.long,
62 + device='cuda')
63 + rej_sampler.num_emitted_tokens = torch.tensor(0,
64 + dtype=torch.long,
65 + device='cuda')
66 + rej_sampler.num_draft_tokens = 0
67 +
68 + collector = AsyncMetricsCollector(rej_sampler)
69 + collector.init_gpu_tensors(rank=rank)
70 + _ = collector.maybe_collect_rejsample_metrics(k=5)
71 + metrics = collector.maybe_collect_rejsample_metrics(k=5)
72 + assert metrics is None
73 +
74 +
75 +def test_noop_until_time():
76 + """Verify metrics aren't collected until enough time passes.
77 + """
78 + rej_sampler = MagicMock()
79 + rej_sampler.num_accepted_tokens = torch.tensor(0,
80 + dtype=torch.long,
81 + device='cuda')
82 + rej_sampler.num_emitted_tokens = torch.tensor(0,
83 + dtype=torch.long,
84 + device='cuda')
85 + rej_sampler.num_draft_tokens = 0
86 +
87 + collect_interval_s = 5.0
88 + timer = MagicMock()
89 + timer.side_effect = [
90 + 0.0, collect_interval_s - 0.1, collect_interval_s - 0.1,
91 + collect_interval_s + 0.1, collect_interval_s + 0.1
92 + ]
93 +
94 + collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
95 + timer=timer,
96 + collect_interval_s=collect_interval_s)
97 + collector.init_gpu_tensors(rank=0)
98 +
99 + _ = collector.maybe_collect_rejsample_metrics(k=5)
100 + metrics = collector.maybe_collect_rejsample_metrics(k=5)
101 + assert metrics is None
102 +
103 + _ = collector.maybe_collect_rejsample_metrics(k=5)
104 + metrics = collector.maybe_collect_rejsample_metrics(k=5)
105 + assert metrics is not None
106 +
107 +
108 +@pytest.mark.parametrize("has_data", [True, False])
109 +def test_initial_metrics_has_correct_values(has_data: bool):
110 + """Test correctness of metrics data.
111 + """
112 + if has_data:
113 + num_accepted_tokens = 103
114 + num_emitted_tokens = 104
115 + num_draft_tokens = 105
116 + else:
117 + num_accepted_tokens = 0
118 + num_emitted_tokens = 0
119 + num_draft_tokens = 0
120 + k = 5
121 +
122 + num_possible_tokens = AsyncMetricsCollector.get_max_num_accepted_tokens(
123 + num_draft_tokens, k)
124 +
125 + rej_sampler = MagicMock()
126 + rej_sampler.num_accepted_tokens = torch.tensor(num_accepted_tokens,
127 + dtype=torch.long,
128 + device='cuda')
129 + rej_sampler.num_emitted_tokens = torch.tensor(num_emitted_tokens,
130 + dtype=torch.long,
131 + device='cuda')
132 + rej_sampler.num_draft_tokens = num_draft_tokens
133 +
134 + collect_interval_s = 5.0
135 + timer = MagicMock()
136 + timer.side_effect = [
137 + 0.0, collect_interval_s + 0.1, collect_interval_s + 0.2
138 + ]
139 +
140 + collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
141 + timer=timer,
142 + collect_interval_s=collect_interval_s)
143 + collector.init_gpu_tensors(rank=0)
144 + _ = collector.maybe_collect_rejsample_metrics(k)
145 + metrics = collector.maybe_collect_rejsample_metrics(k)
146 +
147 + assert metrics.num_spec_tokens == k
148 + assert metrics.accepted_tokens == num_accepted_tokens
149 + assert metrics.draft_tokens == num_draft_tokens
150 + assert metrics.emitted_tokens == num_emitted_tokens
151 +
152 + if has_data:
153 + assert metrics.draft_acceptance_rate == num_accepted_tokens / num_draft_tokens
154 + assert metrics.system_efficiency == num_emitted_tokens / num_possible_tokens
155 + else:
156 + assert math.isnan(metrics.draft_acceptance_rate)
157 + assert math.isnan(metrics.system_efficiency)
tests/spec_decode/test_multi_step_worker.py renamed
+160 -2
@@ -3,14 +3,15 @@ import random
3 import pytest
4 from unittest.mock import MagicMock
5
6 -from vllm.worker.spec_decode.multi_step_worker import MultiStepWorker
6 +from vllm.spec_decode.multi_step_worker import MultiStepWorker, DraftModelTop1Proposer
7 from vllm.worker.worker import Worker
8 from vllm.model_executor.utils import set_random_seed
9 +from vllm.sequence import SamplerOutput
10
11 from .utils import (create_execute_model_data, create_worker,
12 create_seq_group_metadata_from_prompts, zero_kv_cache,
13 patch_execute_model_with_seeds,
13 - assert_logprobs_dict_allclose)
14 + assert_logprobs_dict_allclose, create_batch)
15
16
17 @pytest.mark.parametrize('num_steps', list(range(1, 17)))
@@ -259,3 +260,160 @@ def test_same_output_for_multi_step():
260 multi_step_output_logprobs, single_step_output_logprobs):
261 assert_logprobs_dict_allclose(multi_step_logprobs,
262 single_step_logprobs)
263 +
264 +
265 +@torch.inference_mode()
266 +def test_draft_proposals_full_speculation_len():
267 + """Verify DraftModelTop1Proposer correctly handles case where all sequences
268 + can speculate.
269 + """
270 + k = 10
271 + batch_size = 32
272 + vocab_size = 32_000
273 + device = 'cuda:0'
274 +
275 + draft_worker = MagicMock()
276 + proposer = DraftModelTop1Proposer(
277 + draft_worker=draft_worker,
278 + device=device,
279 + max_model_len=2048,
280 + vocab_size=vocab_size,
281 + )
282 + draft_worker.execute_model_multi_step.return_value = [
283 + SamplerOutput(
284 + outputs=[],
285 + sampled_token_probs=torch.rand(batch_size,
286 + vocab_size,
287 + device=device,
288 + dtype=torch.float32),
289 + sampled_token_ids=torch.randint(low=0,
290 + high=vocab_size,
291 + size=(batch_size, ),
292 + device=device,
293 + dtype=torch.long),
294 + ) for _ in range(k)
295 + ]
296 +
297 + execute_model_data, _, _ = create_batch(batch_size, k)
298 +
299 + proposals = proposer.get_proposals(
300 + **execute_model_data.to_dict(),
301 + max_proposal_len=k,
302 + )
303 +
304 + assert torch.is_tensor(proposals.proposal_token_ids)
305 + assert torch.is_tensor(proposals.proposal_probs)
306 +
307 + assert proposals.proposal_token_ids.shape == torch.Size([batch_size, k])
308 + assert proposals.proposal_probs.shape[:-1] == torch.Size([batch_size, k])
309 +
310 + assert proposals.proposal_lens.shape == torch.Size([batch_size])
311 + assert proposals.proposal_lens.tolist() == [k for _ in range(batch_size)]
312 +
313 +
314 +@torch.inference_mode()
315 +def test_draft_proposals_no_speculations():
316 + """Verify DraftModelTop1Proposer correctly handles case where no sequences
317 + can speculate.
318 + """
319 + k = 10
320 + batch_size = 32
321 + vocab_size = 32_000
322 + device = 'cuda:0'
323 + prompt_len = 10
324 +
325 + draft_worker = MagicMock()
326 + proposer = DraftModelTop1Proposer(
327 + draft_worker=draft_worker,
328 + device=device,
329 + max_model_len=prompt_len + k - 1,
330 + vocab_size=vocab_size,
331 + )
332 +
333 + execute_model_data, _, _ = create_batch(batch_size,
334 + k,
335 + prompt_len=prompt_len)
336 +
337 + proposals = proposer.get_proposals(
338 + **execute_model_data.to_dict(),
339 + max_proposal_len=k,
340 + )
341 +
342 + assert torch.is_tensor(proposals.proposal_token_ids)
343 + assert torch.is_tensor(proposals.proposal_probs)
344 +
345 + assert proposals.proposal_token_ids.shape == torch.Size([0, k])
346 + assert proposals.proposal_probs.shape[:-1] == torch.Size([0, k])
347 +
348 + assert proposals.proposal_lens.shape == torch.Size([batch_size])
349 + assert proposals.proposal_lens.tolist() == [0 for _ in range(batch_size)]
350 +
351 +
352 +@torch.inference_mode()
353 +def test_draft_proposals_mixed_k():
354 + """Verify DraftModelTop1Proposer correctly handles case some sequences can
355 + speculate and some can't.
356 + """
357 + k = 10
358 + batch_size = 32
359 + vocab_size = 32_000
360 + device = 'cuda:0'
361 +
362 + small_prompt_len = 5
363 + long_prompt_len = 10
364 + prev_output_token_len = 20
365 +
366 + expected_num_proposal_seqs = 6
367 + expected_num_no_proposal_seqs = batch_size - expected_num_proposal_seqs
368 +
369 + prompt_len = [
370 + small_prompt_len for _ in range(expected_num_proposal_seqs - 1)
371 + ] + [long_prompt_len
372 + for _ in range(expected_num_no_proposal_seqs)] + [small_prompt_len]
373 +
374 + draft_worker = MagicMock()
375 + proposer = DraftModelTop1Proposer(
376 + draft_worker=draft_worker,
377 + device=device,
378 + max_model_len=long_prompt_len + prev_output_token_len + k - 1,
379 + vocab_size=vocab_size,
380 + )
381 +
382 + draft_worker.execute_model_multi_step.return_value = [
383 + SamplerOutput(
384 + outputs=[],
385 + sampled_token_probs=torch.rand(expected_num_proposal_seqs,
386 + vocab_size,
387 + device=device,
388 + dtype=torch.float32),
389 + sampled_token_ids=torch.randint(
390 + low=0,
391 + high=vocab_size,
392 + size=(expected_num_proposal_seqs, ),
393 + device=device,
394 + dtype=torch.long),
395 + ) for _ in range(k)
396 + ]
397 +
398 + execute_model_data, _, _ = create_batch(
399 + batch_size,
400 + k,
401 + prompt_len=prompt_len,
402 + prev_output_token_len=prev_output_token_len,
403 + )
404 +
405 + proposals = proposer.get_proposals(
406 + **execute_model_data.to_dict(),
407 + max_proposal_len=k,
408 + )
409 +
410 + assert torch.is_tensor(proposals.proposal_token_ids)
411 + assert torch.is_tensor(proposals.proposal_probs)
412 +
413 + assert proposals.proposal_token_ids.shape == torch.Size([batch_size, k])
414 + assert proposals.proposal_probs.shape[:-1] == torch.Size([batch_size, k])
415 +
416 + assert proposals.proposal_lens.shape == torch.Size([batch_size])
417 + assert proposals.proposal_lens.tolist() == [
418 + k for _ in range(expected_num_proposal_seqs - 1)
419 + ] + [0 for _ in range(expected_num_no_proposal_seqs)] + [k]
tests/spec_decode/test_spec_decode_worker.py new
+591
@@ -0,0 +1,591 @@
1 +import torch
2 +import random
3 +import pytest
4 +from unittest.mock import MagicMock
5 +
6 +from vllm.spec_decode.multi_step_worker import MultiStepWorker
7 +from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker, split_num_cache_blocks_evenly
8 +from vllm.spec_decode.interfaces import SpeculativeProposals
9 +from vllm.model_executor.utils import set_random_seed
10 +from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11 +from .utils import mock_worker, create_batch, ExecuteModelData, create_sampler_output_list
12 +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics, AsyncMetricsCollector
13 +
14 +
15 +@pytest.mark.parametrize('k', [1, 2, 6])
16 +@pytest.mark.parametrize('batch_size', [1, 2, 32])
17 +@torch.inference_mode()
18 +def test_correctly_calls_draft_model(k: int, batch_size: int):
19 + """Verify SpecDecodeWorker calls the draft worker with correct
20 + inputs. Everything else is mocked out.
21 + """
22 + draft_worker = mock_worker(cls=MultiStepWorker)
23 + target_worker = mock_worker()
24 + rejection_sampler = MagicMock(spec=RejectionSampler)
25 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
26 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
27 + metrics_collector)
28 +
29 + exception_secret = 'artifical stop'
30 + draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
31 +
32 + execute_model_data, _, _ = create_batch(batch_size, k)
33 +
34 + with pytest.raises(ValueError, match=exception_secret):
35 + worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
36 +
37 + call_args_list = draft_worker.get_spec_proposals.call_args_list
38 + assert len(call_args_list) == 1
39 +
40 + for args, _ in call_args_list:
41 + (seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
42 + blocks_to_copy, actual_k) = args
43 + actual_execute_model_data = ExecuteModelData(seq_group_metadata_list,
44 + blocks_to_swap_in,
45 + blocks_to_swap_out,
46 + blocks_to_copy)
47 + assert actual_execute_model_data == execute_model_data
48 + assert actual_k == k
49 +
50 +
51 +@pytest.mark.parametrize('k', [1, 2, 6])
52 +@pytest.mark.parametrize('batch_size', [1, 2, 32])
53 +@torch.inference_mode()
54 +def test_correctly_calls_target_model(k: int, batch_size: int):
55 + """Verify SpecDecodeWorker calls the target model with correct
56 + inputs. Everything else is mocked out.
57 + """
58 + draft_worker = mock_worker(cls=MultiStepWorker)
59 + target_worker = mock_worker()
60 + rejection_sampler = MagicMock(spec=RejectionSampler)
61 + rejection_sampler.token_id_dtype = torch.int64
62 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
63 +
64 + draft_worker.device = 'cuda'
65 + target_worker.device = 'cuda'
66 +
67 + set_random_seed(1)
68 +
69 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
70 + metrics_collector)
71 + worker.init_model()
72 +
73 + vocab_size = 32_000
74 +
75 + proposal_token_ids = torch.randint(low=0,
76 + high=vocab_size,
77 + size=(batch_size, k),
78 + dtype=torch.int64,
79 + device='cuda')
80 + proposal_probs = torch.rand(batch_size,
81 + k,
82 + vocab_size,
83 + dtype=torch.float32,
84 + device='cuda')
85 + proposal_lens = torch.ones(batch_size, dtype=torch.int64,
86 + device='cuda') * k
87 +
88 + execute_model_data, prompts, prev_output_tokens = create_batch(
89 + batch_size, k)
90 +
91 + draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
92 + proposal_token_ids=proposal_token_ids,
93 + proposal_probs=proposal_probs,
94 + proposal_lens=proposal_lens)
95 +
96 + exception_secret = 'artifical stop'
97 + target_worker.execute_model.side_effect = ValueError(exception_secret)
98 +
99 + with pytest.raises(ValueError, match=exception_secret):
100 + worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
101 +
102 + seen_contexts = []
103 +
104 + call_args_list = target_worker.execute_model.call_args_list
105 + assert len(call_args_list) == 1
106 + for args, kwargs in call_args_list:
107 + target_execute_model_data = ExecuteModelData.from_dict(kwargs)
108 +
109 + assert len(target_execute_model_data.seq_group_metadata_list) == (
110 + k + 1) * batch_size
111 + for seq_group_metadata in (
112 + target_execute_model_data.seq_group_metadata_list):
113 + for seq_data in seq_group_metadata.seq_data.values():
114 + seen_contexts.append(seq_data.get_token_ids())
115 +
116 + expected_seen_contexts = []
117 +
118 + for prompt, prev_generated, draft_tokens in zip(
119 + prompts, prev_output_tokens, proposal_token_ids.tolist()):
120 +
121 + for i in range(len(draft_tokens) + 1):
122 + expected_seen_contexts.append(prompt + prev_generated +
123 + draft_tokens[:i])
124 +
125 + seen_contexts.sort()
126 + expected_seen_contexts.sort()
127 + assert expected_seen_contexts == seen_contexts
128 +
129 +
130 +@pytest.mark.parametrize('k', [1, 2, 6])
131 +@pytest.mark.parametrize('batch_size', [1, 2, 32])
132 +@torch.inference_mode()
133 +def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
134 + """Verify SpecDecodeWorker calls the rejection sampler with
135 + correct inputs. Everything else is mocked out.
136 + """
137 + vocab_size = 32_000
138 +
139 + draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
140 + target_worker = mock_worker(vocab_size=vocab_size)
141 + rejection_sampler = MagicMock(spec=RejectionSampler)
142 + rejection_sampler.token_id_dtype = torch.int64
143 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
144 + draft_worker.device = 'cuda'
145 + target_worker.device = 'cuda'
146 +
147 + set_random_seed(1)
148 +
149 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
150 + metrics_collector)
151 + worker.init_model()
152 +
153 + proposal_token_ids = torch.randint(low=0,
154 + high=vocab_size,
155 + size=(batch_size, k),
156 + dtype=torch.int64,
157 + device='cuda')
158 + proposal_probs = torch.rand(batch_size,
159 + k,
160 + vocab_size,
161 + dtype=torch.float32,
162 + device='cuda')
163 +
164 + proposal_lens = torch.ones(batch_size, dtype=torch.int64,
165 + device='cuda') * k
166 +
167 + execute_model_data, _, _ = create_batch(batch_size, k)
168 +
169 + draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
170 + proposal_token_ids=proposal_token_ids,
171 + proposal_probs=proposal_probs,
172 + proposal_lens=proposal_lens)
173 +
174 + target_token_ids = torch.randint(low=0,
175 + high=vocab_size,
176 + size=(1, batch_size * (k + 1)),
177 + dtype=torch.int64,
178 + device='cuda')
179 + target_token_probs = torch.rand(1,
180 + batch_size * (k + 1),
181 + vocab_size,
182 + dtype=torch.float32,
183 + device='cuda')
184 + target_output = create_sampler_output_list(target_token_ids,
185 + target_token_probs)
186 +
187 + target_worker.execute_model.return_value = target_output[0]
188 +
189 + exception_secret = 'artifical stop'
190 + rejection_sampler.side_effect = ValueError(exception_secret)
191 +
192 + with pytest.raises(ValueError, match=exception_secret):
193 + worker.execute_model(**execute_model_data.to_dict(), num_spec_tokens=k)
194 +
195 + assert len(rejection_sampler.call_args_list) == 1
196 + args, _ = rejection_sampler.call_args_list[0]
197 + (actual_proposal_scores, actual_bonus_token_ids, actual_proposal_probs,
198 + actual_proposal_token_ids) = args
199 +
200 + assert torch.equal(actual_bonus_token_ids,
201 + target_token_ids.reshape(batch_size, k + 1)[:, -1:])
202 + assert torch.equal(
203 + actual_proposal_scores,
204 + target_token_probs.reshape(batch_size, k + 1, -1)[:, :-1])
205 + assert torch.equal(actual_proposal_token_ids, proposal_token_ids)
206 + assert torch.equal(actual_proposal_probs, proposal_probs)
207 +
208 +
209 +@pytest.mark.parametrize('k', [1, 2, 6])
210 +@pytest.mark.parametrize('batch_size', [1, 2, 32])
211 +@torch.inference_mode()
212 +def test_correctly_formats_output(k: int, batch_size: int):
213 + """Verify SpecDecodeWorker formats sampler output correctly.
214 + Everything else is mocked out.
215 + """
216 + vocab_size = 32_000
217 +
218 + draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
219 + target_worker = mock_worker(vocab_size=vocab_size)
220 + rejection_sampler = MagicMock(spec=RejectionSampler)
221 + rejection_sampler.token_id_dtype = torch.int64
222 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
223 + draft_worker.device = 'cuda'
224 + target_worker.device = 'cuda'
225 +
226 + set_random_seed(1)
227 +
228 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
229 + metrics_collector)
230 + worker.init_model()
231 +
232 + proposal_token_ids = torch.randint(low=0,
233 + high=vocab_size,
234 + size=(batch_size, k),
235 + dtype=torch.int64,
236 + device='cuda')
237 + proposal_probs = torch.rand(batch_size,
238 + k,
239 + vocab_size,
240 + dtype=torch.float32,
241 + device='cuda')
242 +
243 + proposal_lens = torch.ones(batch_size, dtype=torch.int64,
244 + device='cuda') * k
245 +
246 + execute_model_data, _, _ = create_batch(batch_size, k)
247 +
248 + draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
249 + proposal_token_ids=proposal_token_ids,
250 + proposal_probs=proposal_probs,
251 + proposal_lens=proposal_lens)
252 +
253 + target_token_ids = torch.randint(low=0,
254 + high=vocab_size,
255 + size=(1, batch_size * (k + 1)),
256 + dtype=torch.int64,
257 + device='cuda')
258 + target_token_probs = torch.rand(1,
259 + batch_size * (k + 1),
260 + vocab_size,
261 + dtype=torch.float32,
262 + device='cuda')
263 + target_output = create_sampler_output_list(target_token_ids,
264 + target_token_probs)
265 +
266 + target_worker.execute_model.return_value = target_output[0]
267 +
268 + rejection_sampler_output = torch.randint(low=0,
269 + high=vocab_size,
270 + size=(batch_size, k + 1),
271 + dtype=torch.int64,
272 + device='cuda')
273 + for i in range(batch_size):
274 + minimum_accepted_tokens = 1
275 + rejection_sampler_output[i][
276 + -random.randint(minimum_accepted_tokens, k + 1):] = -1
277 +
278 + rejection_sampler.return_value = rejection_sampler_output
279 +
280 + output = worker.execute_model(**execute_model_data.to_dict(),
281 + num_spec_tokens=k)
282 +
283 + expected_output = create_sampler_output_list(
284 + rejection_sampler_output.transpose(0, 1), [None for _ in range(k + 1)])
285 +
286 + seq_ids = [
287 + next(iter(seq_group_metadata.seq_data.keys()))
288 + for seq_group_metadata in execute_model_data.seq_group_metadata_list
289 + ]
290 + actual_output_by_seq = {seq_id: [] for seq_id in seq_ids}
291 + expected_output_by_seq = {seq_id: [] for seq_id in seq_ids}
292 +
293 + for step in output:
294 + for seq_group in step:
295 + for sample in seq_group.samples:
296 + seq_id = sample.parent_seq_id
297 + actual_output_by_seq[seq_id].append(sample)
298 +
299 + for step in expected_output:
300 + for seq_group in step:
301 + for sample in seq_group.samples:
302 + seq_id = sample.parent_seq_id
303 + expected_output_by_seq[seq_id].append(sample)
304 +
305 + all_seen_seq_ids = set(
306 + list(actual_output_by_seq.keys()) +
307 + list(expected_output_by_seq.keys()))
308 + for seq_id in all_seen_seq_ids:
309 + actual_by_step = actual_output_by_seq[seq_id]
310 + expected_by_step = expected_output_by_seq[seq_id]
311 +
312 + for i in range(k + 1):
313 + if i >= len(actual_by_step):
314 + assert expected_by_step[i].output_token == -1
315 + continue
316 + assert actual_by_step[i].output_token == expected_by_step[
317 + i].output_token
318 + assert actual_by_step[i].logprobs == expected_by_step[i].logprobs
319 +
320 +
321 +@pytest.mark.parametrize('k', [1, 2])
322 +@pytest.mark.parametrize('batch_size', [1])
323 +@pytest.mark.parametrize('returns_metrics', [True, False])
324 +@torch.inference_mode()
325 +def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
326 + """Verify SpecDecodeWorker collects metrics.
327 + """
328 + vocab_size = 32_000
329 +
330 + draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
331 + target_worker = mock_worker(vocab_size=vocab_size)
332 + rejection_sampler = MagicMock(spec=RejectionSampler)
333 + rejection_sampler.token_id_dtype = torch.int64
334 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
335 + draft_worker.device = 'cuda'
336 + target_worker.device = 'cuda'
337 +
338 + set_random_seed(1)
339 +
340 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
341 + metrics_collector)
342 + worker.init_model()
343 +
344 + proposal_token_ids = torch.randint(low=0,
345 + high=vocab_size,
346 + size=(batch_size, k),
347 + dtype=torch.int64,
348 + device='cuda')
349 + proposal_probs = torch.rand(batch_size,
350 + k,
351 + vocab_size,
352 + dtype=torch.float32,
353 + device='cuda')
354 +
355 + proposal_lens = torch.ones(batch_size, dtype=torch.int64,
356 + device='cuda') * k
357 +
358 + execute_model_data, _, _ = create_batch(batch_size, k)
359 +
360 + draft_worker.get_spec_proposals.return_value = SpeculativeProposals(
361 + proposal_token_ids=proposal_token_ids,
362 + proposal_probs=proposal_probs,
363 + proposal_lens=proposal_lens)
364 +
365 + target_token_ids = torch.randint(low=0,
366 + high=vocab_size,
367 + size=(1, batch_size * (k + 1)),
368 + dtype=torch.int64,
369 + device='cuda')
370 + target_token_probs = torch.rand(1,
371 + batch_size * (k + 1),
372 + vocab_size,
373 + dtype=torch.float32,
374 + device='cuda')
375 + target_output = create_sampler_output_list(target_token_ids,
376 + target_token_probs)
377 +
378 + target_worker.execute_model.return_value = target_output[0]
379 +
380 + rejection_sampler_output = torch.randint(low=0,
381 + high=vocab_size,
382 + size=(batch_size, k + 1),
383 + dtype=torch.int64,
384 + device='cuda')
385 + for i in range(batch_size):
386 + minimum_accepted_tokens = 1
387 + rejection_sampler_output[i][
388 + -random.randint(minimum_accepted_tokens, k + 1):] = -1
389 +
390 + rejection_sampler.return_value = rejection_sampler_output
391 +
392 + mock_rejsample_metrics = MagicMock(
393 + spec=SpecDecodeWorkerMetrics) if returns_metrics else None
394 + metrics_collector.maybe_collect_rejsample_metrics.return_value = mock_rejsample_metrics
395 +
396 + output = worker.execute_model(**execute_model_data.to_dict(),
397 + num_spec_tokens=k)
398 + assert output[0].spec_decode_worker_metrics == mock_rejsample_metrics
399 +
400 + call_args_list = metrics_collector.maybe_collect_rejsample_metrics.call_args_list
401 + assert len(call_args_list) == 1
402 + args, kwargs = call_args_list[0]
403 + assert args[0] == k or kwargs.get('k', -1) == k
404 +
405 +
406 +@pytest.mark.parametrize('k', [0])
407 +@pytest.mark.parametrize('batch_size', [1, 2, 32])
408 +@torch.inference_mode()
409 +def test_k_equals_zero(k: int, batch_size: int):
410 + """Verify that the SpecDecodeWorker calls the draft and target workers
411 + when k is zero. This happens during prefill.
412 + """
413 + draft_worker = mock_worker(cls=MultiStepWorker)
414 + target_worker = mock_worker()
415 + rejection_sampler = MagicMock(spec=RejectionSampler)
416 + rejection_sampler.token_id_dtype = torch.int64
417 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
418 +
419 + draft_worker.device = 'cuda'
420 + target_worker.device = 'cuda'
421 +
422 + set_random_seed(1)
423 +
424 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
425 + metrics_collector)
426 +
427 + execute_model_data, prompts, prev_output_tokens = create_batch(
428 + batch_size, k, prev_output_token_len=0)
429 +
430 + out = worker.execute_model(**execute_model_data.to_dict(),
431 + num_spec_tokens=k)
432 +
433 + assert len(out) == 1, f"expected only one token output when {k=}"
434 + assert out[0].probs is None, "expect gpu tensor references to be None"
435 + assert out[
436 + 0].sampled_tokens is None, "expect gpu tensor references to be None"
437 +
438 + draft_worker.execute_model.assert_called_once_with(
439 + **execute_model_data.to_dict(), return_python_output=False)
440 + target_worker.execute_model.assert_called_once_with(
441 + **execute_model_data.to_dict())
442 +
443 +
444 +@pytest.mark.parametrize('k', [0, 5])
445 +@pytest.mark.parametrize('batch_size', [0])
446 +@torch.inference_mode()
447 +def test_empty_input_batch(k: int, batch_size: int):
448 + """Verify that the SpecDecodeWorker calls the draft and target workers
449 + when the input batch is empty. This can happen if the engine communicates
450 + to the workers information without scheduling a batch.
451 + """
452 + draft_worker = mock_worker(cls=MultiStepWorker)
453 + target_worker = mock_worker()
454 + rejection_sampler = MagicMock(spec=RejectionSampler)
455 + rejection_sampler.token_id_dtype = torch.int64
456 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
457 +
458 + draft_worker.device = 'cuda'
459 + target_worker.device = 'cuda'
460 +
461 + set_random_seed(1)
462 +
463 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
464 + metrics_collector)
465 +
466 + execute_model_data, prompts, prev_output_tokens = create_batch(
467 + batch_size, k, prev_output_token_len=0)
468 +
469 + out = worker.execute_model(**execute_model_data.to_dict(),
470 + num_spec_tokens=k)
471 +
472 + assert len(out) == 1, f"expected only one token output when {k=}"
473 + assert out[0].probs is None, "expect gpu tensor references to be None"
474 + assert out[
475 + 0].sampled_tokens is None, "expect gpu tensor references to be None"
476 +
477 + draft_worker.execute_model.assert_called_once_with(
478 + **execute_model_data.to_dict(), return_python_output=False)
479 + target_worker.execute_model.assert_called_once_with(
480 + **execute_model_data.to_dict())
481 +
482 +
483 +@torch.inference_mode()
484 +def test_init_model():
485 + """Verify SpecDecodeWorker invokes proposer/scorer worker init_model, as
486 + well as other GPU initialization.
487 + """
488 + draft_worker = mock_worker(cls=MultiStepWorker)
489 + target_worker = mock_worker()
490 + rejection_sampler = MagicMock(spec=RejectionSampler)
491 + rejection_sampler.token_id_dtype = torch.int64
492 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
493 +
494 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
495 + metrics_collector)
496 +
497 + worker.init_model()
498 +
499 + draft_worker.init_model.assert_called_once()
500 +
501 + target_worker.init_model.assert_called_once()
502 +
503 + metrics_collector.init_gpu_tensors.assert_called_once()
504 + rejection_sampler.init_gpu_tensors.assert_called_once()
505 +
506 +
507 +@torch.inference_mode()
508 +def test_init_cache_engine():
509 + """Verify SpecDecodeWorker invokes init_cache_engine on proposer/scorer
510 + workers.
511 + """
512 + draft_worker = mock_worker(cls=MultiStepWorker)
513 + target_worker = mock_worker()
514 + rejection_sampler = MagicMock(spec=RejectionSampler)
515 + rejection_sampler.token_id_dtype = torch.int64
516 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
517 +
518 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
519 + metrics_collector)
520 +
521 + cache_config = MagicMock()
522 +
523 + worker.init_cache_engine(cache_config)
524 +
525 + draft_worker.init_cache_engine.assert_called_once_with(cache_config)
526 + target_worker.init_cache_engine.assert_called_once_with(cache_config)
527 +
528 +
529 +@pytest.mark.parametrize('available_gpu_blocks', [1, 1024])
530 +@pytest.mark.parametrize('available_cpu_blocks', [500])
531 +@pytest.mark.parametrize('target_cache_block_size_bytes', [2 * 2 * 4096])
532 +@pytest.mark.parametrize('draft_kv_size_bytes', [0, 2 * 2 * 768, 2 * 2 * 4096])
533 +@torch.inference_mode()
534 +def test_profile_num_available_blocks(available_gpu_blocks: int,
535 + available_cpu_blocks: int,
536 + target_cache_block_size_bytes: int,
537 + draft_kv_size_bytes: int):
538 + """Verify SpecDecodeWorker correctly profiles num available GPU blocks.
539 + Specifically, it should run profiling in the scorer worker, and then evenly
540 + split the blocks between proposer and scorer worker.
541 + """
542 + draft_worker = mock_worker(cls=MultiStepWorker)
543 + target_worker = mock_worker()
544 + rejection_sampler = MagicMock(spec=RejectionSampler)
545 + rejection_sampler.token_id_dtype = torch.int64
546 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
547 +
548 + target_worker.profile_num_available_blocks.return_value = (
549 + available_gpu_blocks, available_cpu_blocks)
550 + target_worker.get_cache_block_size_bytes.return_value = target_cache_block_size_bytes
551 + draft_worker.get_cache_block_size_bytes.return_value = draft_kv_size_bytes
552 +
553 + worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
554 + metrics_collector)
555 +
556 + # These values do not directly impact the adjusted block size calculation,
557 + # so they can be fixed.
558 + gpu_memory_utilization = 0.9
559 + cpu_swap_space = 100
560 + block_size = 16
561 +
562 + num_gpu_blocks, num_cpu_blocks = worker.profile_num_available_blocks(
563 + block_size, gpu_memory_utilization, cpu_swap_space, cache_dtype="auto")
564 +
565 + target_worker.profile_num_available_blocks.assert_called_once_with(
566 + block_size, gpu_memory_utilization, cpu_swap_space, "auto")
567 + assert num_cpu_blocks == available_cpu_blocks
568 +
569 + assert num_gpu_blocks == split_num_cache_blocks_evenly(
570 + target_cache_block_size_bytes, draft_kv_size_bytes,
571 + available_gpu_blocks)
572 +
573 +
574 +@pytest.mark.parametrize('available_gpu_blocks',
575 + list(range(20)) + [1024, 1024**2])
576 +@pytest.mark.parametrize('target_cache_block_size_bytes',
577 + [2 * 2 * 4096, 2 * 2 * 8192])
578 +@pytest.mark.parametrize('draft_kv_size_bytes', [0, 2 * 2 * 768, 2 * 2 * 4096])
579 +@torch.inference_mode()
580 +def test_split_num_cache_blocks_evenly(available_gpu_blocks: int,
581 + target_cache_block_size_bytes: int,
582 + draft_kv_size_bytes: int):
583 + """Verify split_num_cache_blocks_evenly does not exceed original memory
584 + allocation in bytes.
585 + """
586 + num_blocks = split_num_cache_blocks_evenly(target_cache_block_size_bytes,
587 + draft_kv_size_bytes,
588 + available_gpu_blocks)
589 + assert (num_blocks * target_cache_block_size_bytes) + (
590 + num_blocks * draft_kv_size_bytes) <= (available_gpu_blocks *
591 + target_cache_block_size_bytes)
tests/spec_decode/test_utils.py new
+111
@@ -0,0 +1,111 @@
1 +from vllm.spec_decode.util import get_all_seq_ids
2 +from vllm.sequence import SequenceGroupMetadata
3 +from vllm.spec_decode.util import split_batch_by_proposal_len
4 +
5 +import pytest
6 +from unittest.mock import MagicMock
7 +
8 +
9 +def test_get_all_seq_ids():
10 + """Verify get_all_seq_ids extracts all seq ids.
11 + """
12 + expected_seq_ids = list(range(10)) + list(range(100, 110))
13 +
14 + seq_group_metadata_list = [
15 + SequenceGroupMetadata(
16 + request_id=str(seq_id),
17 + is_prompt=True,
18 + seq_data={
19 + seq_id: MagicMock(),
20 + },
21 + sampling_params=MagicMock(),
22 + block_tables={
23 + seq_id: MagicMock(),
24 + },
25 + lora_request=None,
26 + ) for seq_id in expected_seq_ids
27 + ]
28 +
29 + actual_seq_ids = get_all_seq_ids(seq_group_metadata_list)
30 + assert actual_seq_ids == expected_seq_ids
31 +
32 +
33 +@pytest.fixture
34 +def fake_sequence_group_metadata():
35 + seq_ids = list(range(3))
36 + return [
37 + SequenceGroupMetadata(
38 + request_id=str(i),
39 + is_prompt=True,
40 + seq_data={
41 + i: MagicMock(),
42 + },
43 + sampling_params=MagicMock(),
44 + block_tables={
45 + i: MagicMock(),
46 + },
47 + lora_request=None,
48 + ) for i in seq_ids
49 + ]
50 +
51 +
52 +def test_filter_zero_length_proposals(fake_sequence_group_metadata):
53 + proposal_lens = [0, 1, 0]
54 + filtered_groups, indices = split_batch_by_proposal_len(
55 + fake_sequence_group_metadata,
56 + proposal_lens,
57 + select_proposal_len_zero=True)
58 +
59 + expected_groups = [
60 + fake_sequence_group_metadata[0], fake_sequence_group_metadata[2]
61 + ]
62 + expected_indices = [0, 2]
63 +
64 + assert filtered_groups == expected_groups
65 + assert indices == expected_indices
66 +
67 +
68 +def test_filter_non_zero_length_proposals(fake_sequence_group_metadata):
69 + proposal_lens = [0, 1, 2]
70 + filtered_groups, indices = split_batch_by_proposal_len(
71 + fake_sequence_group_metadata,
72 + proposal_lens,
73 + select_proposal_len_zero=False)
74 +
75 + expected_groups = [
76 + fake_sequence_group_metadata[1], fake_sequence_group_metadata[2]
77 + ]
78 + expected_indices = [1, 2]
79 +
80 + assert filtered_groups == expected_groups
81 + assert indices == expected_indices
82 +
83 +
84 +def test_empty_inputs():
85 + filtered_groups, indices = split_batch_by_proposal_len(
86 + [], [], select_proposal_len_zero=True)
87 +
88 + assert filtered_groups == []
89 + assert indices == []
90 +
91 +
92 +def test_all_zero_with_non_zero_filter(fake_sequence_group_metadata):
93 + proposal_lens = [0, 0, 0]
94 + filtered_groups, indices = split_batch_by_proposal_len(
95 + fake_sequence_group_metadata,
96 + proposal_lens,
97 + select_proposal_len_zero=False)
98 +
99 + assert filtered_groups == []
100 + assert indices == []
101 +
102 +
103 +def test_all_non_zero_with_zero_filter(fake_sequence_group_metadata):
104 + proposal_lens = [1, 1, 1]
105 + filtered_groups, indices = split_batch_by_proposal_len(
106 + fake_sequence_group_metadata,
107 + proposal_lens,
108 + select_proposal_len_zero=True)
109 +
110 + assert filtered_groups == []
111 + assert indices == []
tests/spec_decode/utils.py renamed
+96 -19
@@ -1,13 +1,16 @@
1 import torch
2 -from typing import List, Optional, Dict
2 +from typing import List, Optional, Dict, Iterable, Union
3 +from unittest.mock import MagicMock
4
5 from vllm.worker.worker import Worker
6 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7 from vllm.engine.arg_utils import EngineArgs
7 -from vllm.sequence import Logprob, SequenceGroupMetadata, SequenceData
8 +from vllm.sequence import (Logprob, SequenceGroupMetadata, SequenceData,
9 + SamplerOutput, SequenceGroupOutput, SequenceOutput)
10 from vllm.sampling_params import SamplingParams
11 from vllm.worker.cache_engine import CacheEngine
12 from vllm.model_executor.utils import set_random_seed
13 +from itertools import count
14 from dataclasses import dataclass, fields
15
16
@@ -24,6 +27,11 @@ class ExecuteModelData:
27 return dict(
28 (field.name, getattr(self, field.name)) for field in fields(self))
29
30 + @classmethod
31 + def from_dict(cls, d):
32 + cleaned = dict((field.name, d[field.name]) for field in fields(cls))
33 + return cls(**cleaned)
34 +
35
36 def round_up_to_next_block(seq_len: int, block_size: int) -> int:
37 return (seq_len + block_size - 1) // block_size
@@ -50,6 +58,21 @@ def create_execute_model_data(
58 )
59
60
61 +def mock_worker(cls=None,
62 + vocab_size: int = 30_000,
63 + max_model_len: int = 2048,
64 + rank: int = 0) -> MagicMock:
65 + if cls is None:
66 + cls = Worker
67 +
68 + worker = MagicMock(spec=cls)
69 + worker.vocab_size = vocab_size
70 + worker.max_model_len = max_model_len
71 + worker.rank = rank
72 + worker.device = 'cuda:0'
73 + return worker
74 +
75 +
76 def patch_execute_model_with_seeds(worker: Worker, rand_seeds: List[int]):
77 seed_iter = iter(rand_seeds)
78 original_execute_model = worker.execute_model
@@ -117,25 +140,12 @@ def create_seq_group_metadata_from_prompts(
140 block_size: int,
141 final_seq_lens: List[int],
142 continuations: Optional[List[List[int]]] = None,
120 - num_tokens_processed: Optional[List[int]] = None,
143 seq_ids: Optional[List[int]] = None,
144 ) -> List[SequenceGroupMetadata]:
145
146 if continuations is None:
147 continuations = [[] for _ in prompts]
148
127 - if num_tokens_processed is None:
128 - # Default to 1 token missing from kv cache for generation sequences.
129 - num_tokens_processed = []
130 - for continuation, prompt in zip(continuations, prompts):
131 - # If prefill, then default to zero tokens processed.
132 - if not continuation:
133 - num_tokens_processed.append(0)
134 - else:
135 - # If generation, then default to all but one tokens processed.
136 - num_tokens_processed.append(
137 - len(continuation) + len(prompt) - 1)
138 -
149 if seq_ids is None:
150 seq_ids = list(i for i, _ in enumerate(prompts))
151
@@ -155,13 +165,15 @@ def create_seq_group_metadata_from_prompts(
165 is_prompt=len(cont_token_ids) == 0,
166 seq_data={
167 i:
158 - SequenceData(prompt_token_ids=prompt_token_ids[:] +
159 - cont_token_ids[:])
168 + SequenceData(
169 + prompt_token_ids=prompt_token_ids[:],
170 + output_token_ids=cont_token_ids[:],
171 + ),
172 },
173 sampling_params=SamplingParams(temperature=0.0, ),
174 block_tables={i: block_allocations[i][:]},
163 - ) for i, (prompt_token_ids, cont_token_ids, num_tokens_saved) in
164 - enumerate(zip(prompts, continuations, num_tokens_processed))
175 + ) for i, (prompt_token_ids,
176 + cont_token_ids) in enumerate(zip(prompts, continuations))
177 ]
178
179
@@ -178,3 +190,68 @@ def assert_logprobs_dict_allclose(
190 expected = torch.tensor(
191 single_step_expected_logprobs[token_id].logprob)
192 assert torch.allclose(actual, expected)
193 +
194 +
195 +def create_sampler_output_list(
196 + token_ids: torch.Tensor,
197 + probs: Iterable[Optional[torch.Tensor]],
198 + seq_ids: Optional[List[int]] = None) -> List[SamplerOutput]:
199 + num_steps, batch_size = token_ids.shape
200 + token_ids_by_step = token_ids.tolist()
201 +
202 + if seq_ids is None:
203 + seq_ids = list(range(batch_size))
204 +
205 + return [
206 + SamplerOutput(outputs=[
207 + SequenceGroupOutput(
208 + samples=[
209 + SequenceOutput(
210 + output_token=token_id,
211 + parent_seq_id=seq_ids[seq_index],
212 + logprobs={token_id: 0},
213 + )
214 + ],
215 + prompt_logprobs=None,
216 + ) for seq_index, token_id in enumerate(token_ids_by_step[step])
217 + ],
218 + sampled_token_probs=probs[step],
219 + sampled_token_ids=token_ids[step])
220 + for step in range(num_steps)
221 + ]
222 +
223 +
224 +def create_batch(batch_size,
225 + k,
226 + prompt_len: Union[int, List[int]] = 10,
227 + prev_output_token_len: int = 10,
228 + seq_ids: Optional[List[int]] = None,
229 + num_gpu_blocks: Optional[int] = None,
230 + block_size: Optional[int] = None):
231 + if block_size is None:
232 + block_size = 8
233 +
234 + if num_gpu_blocks is None:
235 + num_gpu_blocks = 2048 // block_size
236 +
237 + iterator = count()
238 +
239 + if isinstance(prompt_len, int):
240 + prompt_lens = [prompt_len for _ in range(batch_size)]
241 + else:
242 + prompt_lens = prompt_len
243 +
244 + prompts = [[next(iterator) for _ in range(p_len)] for p_len in prompt_lens]
245 + prev_output_tokens = [[
246 + next(iterator) for _ in range(prev_output_token_len)
247 + ] for _ in range(batch_size)]
248 + final_seq_lens = [
249 + len(prompt) + len(prev_output_token) + k + 1
250 + for prompt, prev_output_token in zip(prompts, prev_output_tokens)
251 + ]
252 +
253 + execute_model_data = create_execute_model_data(
254 + create_seq_group_metadata_from_prompts(prompts, num_gpu_blocks,
255 + block_size, final_seq_lens,
256 + prev_output_tokens, seq_ids), )
257 + return execute_model_data, prompts, prev_output_tokens
tests/test_sequence.py new
+50
@@ -0,0 +1,50 @@
1 +import pytest
2 +
3 +from vllm.sequence import SequenceGroupOutput, SamplerOutput, SequenceOutput
4 +
5 +
6 +@pytest.fixture
7 +def sample_outputs():
8 + return [
9 + SequenceGroupOutput(samples=[
10 + SequenceOutput(parent_seq_id=0, output_token=i, logprobs={})
11 + ],
12 + prompt_logprobs=None) for i in range(5)
13 + ]
14 +
15 +
16 +@pytest.fixture
17 +def sampler_output(sample_outputs):
18 + return SamplerOutput(outputs=sample_outputs)
19 +
20 +
21 +def test_sampler_output_initialization(sampler_output, sample_outputs):
22 + assert len(sampler_output) == len(sample_outputs)
23 + assert sampler_output.sampled_token_probs is None
24 + assert sampler_output.sampled_token_ids is None
25 + assert sampler_output.spec_decode_worker_metrics is None
26 +
27 +
28 +def test_sampler_output_getitem(sampler_output, sample_outputs):
29 + assert sampler_output[2] == sample_outputs[2]
30 +
31 +
32 +def test_sampler_output_setitem(sampler_output):
33 + new_output = SequenceGroupOutput(samples=[
34 + SequenceOutput(parent_seq_id=0, output_token=99, logprobs={})
35 + ],
36 + prompt_logprobs=None)
37 + sampler_output[2] = new_output
38 + assert sampler_output[2] == new_output
39 +
40 +
41 +def test_sampler_output_len(sampler_output, sample_outputs):
42 + assert len(sampler_output) == len(sample_outputs)
43 +
44 +
45 +def test_sampler_output_eq(sample_outputs):
46 + sampler_output1 = SamplerOutput(outputs=sample_outputs)
47 + sampler_output2 = SamplerOutput(outputs=sample_outputs.copy())
48 + sampler_output3 = SamplerOutput(outputs=sample_outputs[:-1])
49 + assert sampler_output1 == sampler_output2
50 + assert sampler_output1 != sampler_output3
vllm/model_executor/layers/rejection_sampler.py
+8 -2
@@ -21,8 +21,6 @@ class RejectionSampler(nn.Module):
21 nontrivial latency.
22 """
23 super().__init__()
24 - self.probs_dtype = torch.float32
25 - self.token_id_dtype = torch.int64
24 self._strict_mode = strict_mode
25
26 # NOTE: A "bonus token" is accepted iff all proposal tokens are
@@ -44,6 +42,14 @@ class RejectionSampler(nn.Module):
42 dtype=torch.long,
43 device=device)
44
45 + @property
46 + def probs_dtype(self):
47 + return torch.float32
48 +
49 + @property
50 + def token_id_dtype(self):
51 + return torch.int64
52 +
53 def forward(
54 self,
55 target_probs: torch.Tensor,
vllm/model_executor/layers/sampler.py
+1 -1
@@ -587,4 +587,4 @@ def _build_sampler_output(
587 SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
588 sampler_output.append(
589 SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
590 - return sampler_output
590 + return SamplerOutput(outputs=sampler_output)
vllm/sequence.py
+50 -5
@@ -2,12 +2,16 @@
2 import copy
3 import enum
4 from dataclasses import dataclass
5 -from typing import Dict, List, Optional, Union
5 +from typing import Dict, List, Optional, Union, TYPE_CHECKING
6
7 from vllm.block import LogicalTokenBlock
8 from vllm.sampling_params import SamplingParams
9 from vllm.lora.request import LoRARequest
10
11 +if TYPE_CHECKING:
12 + import torch
13 + from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
14 +
15
16 @dataclass
17 class Logprob:
@@ -81,6 +85,8 @@ class SequenceData:
85
86 Args:
87 prompt_token_ids: The token IDs of the prompt.
88 + output_token_ids: The token IDs of the output. Set to an empty list if
89 + None.
90
91 Attributes:
92 prompt_token_ids: The token IDs of the prompt.
@@ -91,9 +97,13 @@ class SequenceData:
97 def __init__(
98 self,
99 prompt_token_ids: List[int],
100 + output_token_ids: Optional[List[int]] = None,
101 ) -> None:
102 + if output_token_ids is None:
103 + output_token_ids = []
104 +
105 self.prompt_token_ids = prompt_token_ids
96 - self.output_token_ids: List[int] = []
106 + self.output_token_ids = output_token_ids
107 self.cumulative_logprob = 0.0
108
109 def append_token_id(self, token_id: int, logprob: float) -> None:
@@ -117,6 +127,12 @@ class SequenceData:
127 return self.prompt_token_ids[-1]
128 return self.output_token_ids[-1]
129
130 + def get_prompt_token_ids(self) -> int:
131 + return self.prompt_token_ids
132 +
133 + def get_output_token_ids(self) -> int:
134 + return self.output_token_ids
135 +
136 def __repr__(self) -> str:
137 return (f"SequenceData("
138 f"prompt_token_ids={self.prompt_token_ids}, "
@@ -506,6 +522,35 @@ class SequenceGroupOutput:
522 and self.prompt_logprobs == other.prompt_logprobs)
523
524
509 -# For each sequence group, we generate a list of SequenceOutput object,
510 -# each of which contains one possible candidate for the next token.
511 -SamplerOutput = List[SequenceGroupOutput]
525 +@dataclass
526 +class SamplerOutput:
527 + """For each sequence group, we generate a list of SequenceOutput object,
528 + each of which contains one possible candidate for the next token.
529 +
530 + This datastructure implements methods so it can be used like a list, but
531 + also has optional fields for device tensors.
532 + """
533 +
534 + outputs: List[SequenceGroupOutput]
535 +
536 + # On-device tensor containing probabilities of each token.
537 + sampled_token_probs: Optional["torch.Tensor"] = None
538 +
539 + # On-device tensor containing the sampled token ids.
540 + sampled_token_ids: Optional["torch.Tensor"] = None
541 +
542 + # Spec decode metrics populated by workers.
543 + spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
544 +
545 + def __getitem__(self, idx: int):
546 + return self.outputs[idx]
547 +
548 + def __setitem__(self, idx: int, value):
549 + self.outputs[idx] = value
550 +
551 + def __len__(self):
552 + return len(self.outputs)
553 +
554 + def __eq__(self, other: object):
555 + return isinstance(other,
556 + self.__class__) and self.outputs == other.outputs
vllm/spec_decode/batch_expansion.py new
+351
@@ -0,0 +1,351 @@
1 +from typing import Iterator, List, Tuple, Optional, Dict
2 +from itertools import chain, count
3 +
4 +import torch
5 +
6 +from vllm.sequence import (SamplerOutput, SequenceGroupMetadata, SequenceData)
7 +from vllm.worker.worker import Worker
8 +from vllm.spec_decode.util import nvtx_range, sampler_output_to_torch, get_all_seq_ids, split_batch_by_proposal_len
9 +from vllm.spec_decode.interfaces import SpeculativeScorer, SpeculativeProposals, SpeculativeScores
10 +
11 +SeqId = int
12 +TargetSeqId = int
13 +TokenId = int
14 +
15 +
16 +class BatchExpansionTop1Scorer(SpeculativeScorer):
17 + """Implements a speculative scorer that uses batch expansion to get
18 + probabilities of speculative tokens according to the scoring model.
19 +
20 + Batch expansion converts a list of sequences and multiple query positions
21 + to a new batch of sequences, each with a single query position. This allows
22 + for MQA-like scoring in speculative decoding without requiring an MQA
23 + kernel.
24 +
25 + It is strictly less efficient than MQA scoring.
26 +
27 + It only supports scoring the top1 proposal tokens of the proposer, instead
28 + of topk/tree.
29 + """
30 +
31 + def __init__(self, scorer_worker: Worker, device: str, vocab_size: int):
32 + self._scorer_worker = scorer_worker
33 + self._device = device
34 + self._vocab_size = vocab_size
35 +
36 + @nvtx_range("BatchExpansionTop1Scorer.score_proposals")
37 + def score_proposals(
38 + self,
39 + seq_group_metadata_list: List[SequenceGroupMetadata],
40 + blocks_to_swap_in: Optional[Dict[int, int]],
41 + blocks_to_swap_out: Optional[Dict[int, int]],
42 + blocks_to_copy: Optional[Dict[int, List[int]]],
43 + k: int,
44 + proposals: SpeculativeProposals,
45 + ) -> SpeculativeScores:
46 + """Score the proposed tokens via the scorer model.
47 +
48 + This converts each input sequence to a set of k+1 target sequences. The
49 + target sequences have the unique continuations to be scored and a
50 + unique sequence ID that is different from all input sequence ids.
51 +
52 + If a speculative sequence length would exceed the max model length, then
53 + no speculation is produced for that sequence.
54 +
55 + Args:
56 + seq_group_metadata_list: The input sequence group metadata.
57 + blocks_to_swap_in: This is passed to the worker during scoring.
58 + blocks_to_swap_out: This is passed to the worker during scoring.
59 + blocks_to_copy: This is passed to the worker during scoring.
60 + k: The fixed proposal length.
61 + proposals: The speculative proposals to score.
62 + Returns:
63 + SpeculativeScores: The scores of each speculative token, along with
64 + which sequences were ignored during scoring.
65 + """
66 +
67 + # TODO(cade) perform this on GPU to remove blocking call.
68 + proposal_lens_list = proposals.proposal_lens.tolist()
69 + proposal_token_ids_list = proposals.proposal_token_ids.tolist()
70 +
71 + spec_indices, non_spec_indices, target_seq_group_metadata_list, num_scoring_tokens = self._expand_batch(
72 + seq_group_metadata_list=seq_group_metadata_list,
73 + proposal_token_ids_list=proposal_token_ids_list,
74 + proposal_lens_list=proposal_lens_list,
75 + )
76 +
77 + target_sampler_output = self._scorer_worker.execute_model(
78 + seq_group_metadata_list=target_seq_group_metadata_list,
79 + blocks_to_swap_in=blocks_to_swap_in,
80 + blocks_to_swap_out=blocks_to_swap_out,
81 + blocks_to_copy=blocks_to_copy,
82 + return_python_output=False)
83 +
84 + all_tokens, all_probs = self._contract_batch(
85 + original_bs=len(seq_group_metadata_list),
86 + target_sampler_output=target_sampler_output,
87 + proposals=proposals,
88 + num_scoring_tokens=num_scoring_tokens,
89 + non_spec_indices=non_spec_indices,
90 + spec_indices=spec_indices,
91 + k=k,
92 + )
93 +
94 + return SpeculativeScores(
95 + probs=all_probs,
96 + token_ids=all_tokens,
97 + )
98 +
99 + def _expand_batch(
100 + self,
101 + seq_group_metadata_list: List[SequenceGroupMetadata],
102 + proposal_token_ids_list: List[TokenId],
103 + proposal_lens_list: List[int],
104 + ) -> Tuple[List[int], List[int], List[SequenceGroupMetadata], int]:
105 + """Given the input sequences and potentially multiple corresponding
106 + proposal tokens, create a new batch where each sequence has a single
107 + query token.
108 + """
109 +
110 + # vLLM currently only supports proposal lens equal to zero or the batch
111 + # proposal len. This adds some complexity (splitting the batch into spec
112 + # and non spec sequences) and should be removed in the future. It can be
113 + # done by supporting per-sequence proposal lens.
114 + spec_seqs, spec_indices = split_batch_by_proposal_len(
115 + seq_group_metadata_list,
116 + proposal_lens_list,
117 + select_proposal_len_zero=False)
118 + non_spec_seqs, non_spec_indices = split_batch_by_proposal_len(
119 + seq_group_metadata_list,
120 + proposal_lens_list,
121 + select_proposal_len_zero=True)
122 +
123 + target_seq_group_metadata_list = self._create_scoring_model_input(
124 + spec_seqs, proposal_token_ids_list)
125 + num_scoring_tokens = len(target_seq_group_metadata_list)
126 + target_seq_group_metadata_list.extend(non_spec_seqs)
127 +
128 + return spec_indices, non_spec_indices, target_seq_group_metadata_list, num_scoring_tokens
129 +
130 + def _contract_batch(self, original_bs: int,
131 + target_sampler_output: List[SamplerOutput],
132 + proposals: SpeculativeProposals,
133 + num_scoring_tokens: int, non_spec_indices: List[int],
134 + spec_indices: List[int],
135 + k: int) -> Tuple[torch.Tensor, torch.Tensor]:
136 + """Contract the expanded batch back into its original size.
137 + This maps the scores of speculative tokens back to their original
138 + sequences.
139 + """
140 + (target_token_ids, target_probs, non_spec_target_token_ids,
141 + non_spec_target_probs) = self._split_scoring_output(
142 + target_sampler_output, num_scoring_tokens)
143 +
144 + # Map distinct sequences used to score each token
145 + # of shape [batch_size * k + 1] back to [batch_size, k + 1].
146 + batch_size, k = proposals.proposal_token_ids.shape
147 +
148 + target_token_ids = target_token_ids.squeeze().reshape(
149 + batch_size, k + 1)
150 + target_probs = target_probs.squeeze().reshape(batch_size, k + 1,
151 + self._vocab_size)
152 +
153 + all_tokens = torch.full(size=(original_bs, k + 1),
154 + fill_value=-1,
155 + device=self._device,
156 + dtype=torch.long)
157 + all_probs = torch.zeros(original_bs,
158 + k + 1,
159 + self._vocab_size,
160 + device=self._device,
161 + dtype=torch.float32)
162 +
163 + if non_spec_indices:
164 + all_tokens[non_spec_indices, 0] = non_spec_target_token_ids
165 + all_probs[non_spec_indices, :1, :] = non_spec_target_probs
166 +
167 + if spec_indices:
168 + all_tokens[spec_indices] = target_token_ids
169 + all_probs[spec_indices] = target_probs
170 +
171 + return all_tokens, all_probs
172 +
173 + def _create_scoring_model_input(
174 + self,
175 + seq_group_metadata_list: List[SequenceGroupMetadata],
176 + proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k]
177 + ) -> List[SequenceGroupMetadata]:
178 + """Given the original input sequences and proposed tokens from the draft
179 + model, create a list of target sequences that can be used for scoring.
180 + """
181 +
182 + if not seq_group_metadata_list:
183 + return []
184 +
185 + target_seq_ids_iter = self._create_target_seq_id_iterator(
186 + get_all_seq_ids(seq_group_metadata_list))
187 +
188 + target_seq_group_metadata = list(
189 + chain.from_iterable(
190 + self._create_target_seq_group_metadata(
191 + seq_group_metadata,
192 + proposal_token_ids,
193 + i,
194 + target_seq_ids_iter,
195 + ) for i, seq_group_metadata in enumerate(
196 + seq_group_metadata_list)))
197 +
198 + return target_seq_group_metadata
199 +
200 + def _create_target_seq_group_metadata(
201 + self,
202 + input_seq_group_metadata: SequenceGroupMetadata,
203 + proposal_token_ids: List[TokenId], # shape: [batch_size, k]
204 + batch_index: int,
205 + target_seq_ids_iter: Iterator[TargetSeqId],
206 + ) -> List[SequenceGroupMetadata]:
207 + """Given an input sequence group metadata and a list of draft tokens,
208 + create a list of target SequenceGroupMetadata, one for each
209 + token id that needs to be scored.
210 +
211 + Naive speculative decoding requires K target model scores, one for each
212 + draft model token. However one can add a bonus token such that if each
213 + token is accepted, then a final token may be sampled from the model.
214 + This function creates K+1 target SequenceGroupMetadata to take
215 + advantage of the bonus token.
216 + """
217 + assert not input_seq_group_metadata.is_prompt, (
218 + "Speculating on "
219 + "prompts not yet supported")
220 + assert len(input_seq_group_metadata.seq_data) == 1, (
221 + "Beam search "
222 + "not supported in speculative decoding")
223 + input_seq_id = next(iter(input_seq_group_metadata.seq_data.keys()))
224 +
225 + token_ids_to_score = self._get_token_ids_to_score(
226 + proposal_token_ids[batch_index])
227 +
228 + target_seq_group_metadata_list: List[SequenceGroupMetadata] = []
229 + for token_ids in token_ids_to_score:
230 + target_seq_group_metadata_list.append(
231 + self._create_single_target_seq_group_metadata(
232 + input_seq_group_metadata,
233 + input_seq_id,
234 + next(target_seq_ids_iter),
235 + token_ids,
236 + ))
237 +
238 + return target_seq_group_metadata_list
239 +
240 + def _create_single_target_seq_group_metadata(
241 + self,
242 + seq_group_metadata: SequenceGroupMetadata,
243 + seq_id: SeqId,
244 + target_seq_id: TargetSeqId,
245 + token_ids: List[TokenId],
246 + ) -> SequenceGroupMetadata:
247 + """Create a single target SequenceGroupMetadata.
248 +
249 + Args:
250 + seq_group_metadata: The metadata for the input sequence.
251 + seq_id: The input sequence ID.
252 + target_seq_id: The corresponding target sequence ID.
253 + token_ids: The list of token ids that are to be appended to the
254 + input sequence.
255 + """
256 + seq_data = seq_group_metadata.seq_data[seq_id]
257 + prompt_token_ids = seq_data.get_prompt_token_ids()
258 + new_output_token_ids = [*seq_data.get_output_token_ids(), *token_ids]
259 +
260 + return SequenceGroupMetadata(
261 + request_id=seq_group_metadata.request_id,
262 + is_prompt=seq_group_metadata.is_prompt,
263 + seq_data={
264 + target_seq_id:
265 + SequenceData(
266 + prompt_token_ids=prompt_token_ids,
267 + output_token_ids=new_output_token_ids,
268 + ),
269 + },
270 + sampling_params=seq_group_metadata.sampling_params,
271 + block_tables={
272 + target_seq_id: seq_group_metadata.block_tables[seq_id],
273 + },
274 + lora_request=None,
275 + )
276 +
277 + def _split_scoring_output(
278 + self, sampler_output: SamplerOutput, num_scoring_tokens: int
279 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
280 + """Split the target model output into speculative and non-speculative
281 + output.
282 + """
283 +
284 + # vLLM currently only supports proposal lens equal to zero or the batch
285 + # proposal len. This adds some complexity (splitting the batch into spec
286 + # and non spec sequences) and should be removed in the future. It can be
287 + # done by supporting per-sequence proposal lens.
288 + #
289 + # First samples are from speculative scoring, latter samples are non-
290 + # speculative samples.
291 + split_sizes = [
292 + num_scoring_tokens,
293 + sampler_output.sampled_token_ids.numel() - num_scoring_tokens
294 + ]
295 + (spec_probs, non_spec_probs
296 + ) = sampler_output.sampled_token_probs.split(split_sizes)
297 + (spec_sampled_tokens, non_spec_sampled_tokens
298 + ) = sampler_output.sampled_token_ids.flatten().split(split_sizes)
299 +
300 + # Convert scores to tensors.
301 + sampler_output.sampled_token_probs = spec_probs
302 + sampler_output.sampled_token_ids = spec_sampled_tokens
303 + target_token_ids, target_probs = sampler_output_to_torch(
304 + [sampler_output])
305 +
306 + # Convert non-speculative output tokens to tensors.
307 + sampler_output.sampled_token_probs = non_spec_probs
308 + sampler_output.sampled_token_ids = non_spec_sampled_tokens
309 + non_spec_target_token_ids, non_spec_target_probs = sampler_output_to_torch(
310 + [sampler_output])
311 +
312 + return target_token_ids, target_probs, non_spec_target_token_ids, non_spec_target_probs
313 +
314 + def _create_target_seq_id_iterator(
315 + self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
316 + """Create an iterator for creating target sequence ids.
317 + Target sequence ids are distinct from sequence ids because we create a
318 + distinct target sequence id for each proposal token to be scored.
319 +
320 + This implementation increments a counter starting at 1 + max of all
321 + provided input sequence ids.
322 + """
323 + return count(start=max(seq_ids) + 1)
324 +
325 + def _get_token_ids_to_score(
326 + self,
327 + full_spec_token_ids: List[TokenId] # shape: [k]
328 + ) -> List[List[TokenId]]:
329 + """Given an int tensor of proposal token ids, return a list of
330 + token ids that should be scored.
331 +
332 + Returns k+1 output lists. The additional one is used for generating the
333 + bonus token.
334 +
335 + Example:
336 + Input: [0, 1, 2, 3] (k=4)
337 + Output: (k+1 lists)
338 + []
339 + [0]
340 + [0, 1]
341 + [0, 1, 2]
342 + [0, 1, 2, 3]
343 + """
344 + empty_token_ids = []
345 +
346 + token_ids_to_score = [empty_token_ids]
347 + token_ids_to_score.extend([
348 + full_spec_token_ids[:i + 1]
349 + for i in range(len(full_spec_token_ids))
350 + ])
351 + return token_ids_to_score
vllm/spec_decode/interfaces.py new
+77
@@ -0,0 +1,77 @@
1 +from typing import List, Tuple, Optional, Dict
2 +from dataclasses import dataclass
3 +from abc import ABC, abstractmethod
4 +
5 +import torch
6 +
7 +from vllm.sequence import SequenceGroupMetadata
8 +
9 +
10 +@dataclass
11 +class SpeculativeProposals:
12 + """Datastructure used to represent proposal tokens from some proposer. It
13 + also tracks how many speculative tokens each sequence has.
14 + """
15 +
16 + # Speculative proposal tokens.
17 + proposal_token_ids: torch.Tensor
18 +
19 + # Probabilities of the proposal tokens according to the proposer.
20 + proposal_probs: torch.Tensor
21 +
22 + # The valid length of each proposal; can be zero.
23 + proposal_lens: torch.Tensor
24 +
25 + def __repr__(self):
26 + return (f"SpeculativeProposals("
27 + f"proposal_token_ids={self.proposal_token_ids.shape}, "
28 + f"proposal_probs={self.proposal_probs.shape}, "
29 + f"proposal_lens={self.proposal_lens.shape})")
30 +
31 +
32 +@dataclass
33 +class SpeculativeScores:
34 + """Datastructure used to represent the scores of speculative tokens
35 + according to the scoring model.
36 + """
37 +
38 + # Probabilities of the speculative tokens according to the scoring model.
39 + probs: torch.Tensor
40 +
41 + # Token ids sampled from the scoring model. Used for speculative bonus
42 + # tokens and also non-speculative normal decoding.
43 + token_ids: torch.Tensor
44 +
45 + def __repr__(self):
46 + return (f"SpeculativeScores("
47 + f"probs={self.probs.shape}, "
48 + f"token_ids={self.token_ids.shape})")
49 +
50 +
51 +class SpeculativeProposer(ABC):
52 +
53 + @abstractmethod
54 + def get_proposals(
55 + self,
56 + seq_group_metadata_list: List[SequenceGroupMetadata],
57 + blocks_to_swap_in: Dict[int, int],
58 + blocks_to_swap_out: Dict[int, int],
59 + blocks_to_copy: Dict[int, List[int]],
60 + max_proposal_len: int,
61 + ) -> SpeculativeProposals:
62 + raise NotImplementedError
63 +
64 +
65 +class SpeculativeScorer(ABC):
66 +
67 + @abstractmethod
68 + def score_proposals(
69 + self,
70 + seq_group_metadata_list: List[SequenceGroupMetadata],
71 + blocks_to_swap_in: Optional[Dict[int, int]],
72 + blocks_to_swap_out: Optional[Dict[int, int]],
73 + blocks_to_copy: Optional[Dict[int, List[int]]],
74 + k: int,
75 + proposals: SpeculativeProposals,
76 + ) -> Tuple[torch.Tensor, torch.Tensor]:
77 + raise NotImplementedError
vllm/spec_decode/metrics.py new
+174
@@ -0,0 +1,174 @@
1 +import torch
2 +from dataclasses import dataclass
3 +from vllm.model_executor.layers.rejection_sampler import RejectionSampler
4 +from typing import Optional
5 +from vllm.utils import in_wsl
6 +import time
7 +from typing import Callable
8 +
9 +
10 +@dataclass
11 +class SpecDecodeWorkerMetrics:
12 + """Dataclass holding metrics emitted from the spec decode worker.
13 + """
14 +
15 + # The empirical acceptance rate of the proposal method on a per-token basis.
16 + # This is useful for evaluating how well the proposal method aligns with the
17 + # scoring method.
18 + draft_acceptance_rate: float
19 +
20 + # The empirical efficiency, measured as the number of tokens emitted by the
21 + # system divided by the number of tokens that could be emitted by the system
22 + # if the proposal method were perfect.
23 + system_efficiency: float
24 +
25 + # The number of speculative tokens produced by the proposal method.
26 + draft_tokens: int
27 +
28 + # The number of tokens emitted by the entire system.
29 + emitted_tokens: int
30 +
31 + # The number of tokens accepted by the scoring model and verification
32 + # routine, e.g. Llama2-70B and lossless rejection sampling.
33 + #
34 + # NOTE: Any token accepted by the verification routine is considered
35 + # accepted (regardless of if the speculative prefix is also accepted). The
36 + # user will usually see less accepted tokens. This metric is helpful when
37 + # evaluating alignment of the proposal method with the scoring model.
38 + accepted_tokens: int
39 +
40 + # The number of speculative tokens per sequence.
41 + num_spec_tokens: int
42 +
43 +
44 +Timer = Callable[[], float]
45 +
46 +
47 +class AsyncMetricsCollector:
48 + """Class which copies rejection sampler metrics from the device to CPU on a
49 + non-default Torch stream.
50 + """
51 +
52 + def __init__(self,
53 + rejection_sampler: RejectionSampler,
54 + timer: Optional[Timer] = None,
55 + collect_interval_s: float = 5.0):
56 + self._rejection_sampler = rejection_sampler
57 + self._timer = time.time if timer is None else timer
58 +
59 + self._rank: Optional[int] = None
60 +
61 + # We don't have a device set yet.
62 + self._copy_stream: Optional[torch.cuda.Stream] = None
63 +
64 + self._in_flight_copy: Optional[torch.cuda.Event] = None
65 +
66 + pin_memory = not in_wsl()
67 + self._aggregate_num_accepted_tokens = torch.tensor(
68 + 0, dtype=torch.long, device="cpu", pin_memory=pin_memory)
69 + self._aggregate_num_emitted_tokens = torch.tensor(
70 + 0, dtype=torch.long, device="cpu", pin_memory=pin_memory)
71 + self._aggregate_num_draft_tokens = 0
72 +
73 + self._rejsample_metrics_collect_interval_s = collect_interval_s
74 + self._last_metrics_collect_time = self._timer()
75 +
76 + def init_gpu_tensors(self, rank: int) -> None:
77 + self._rank = rank
78 + self._copy_stream = torch.cuda.Stream()
79 +
80 + def maybe_collect_rejsample_metrics(
81 + self, k: int) -> Optional[SpecDecodeWorkerMetrics]:
82 +
83 + # If a copy was initiated in the previous call, collect and return.
84 + if self._in_flight_copy is not None:
85 + ready_event = self._in_flight_copy
86 + self._in_flight_copy = None
87 + return self._collect_rejsample_metrics(k, ready_event)
88 +
89 + # Otherwise, check if we should start a new copy.
90 + if self._should_collect_rejsample_metrics(self._timer()):
91 + assert self._in_flight_copy is None
92 + self._in_flight_copy = self._copy_rejsample_metrics_async()
93 +
94 + return None
95 +
96 + def _should_collect_rejsample_metrics(self, now: float) -> bool:
97 + """Return whether or not this iteration should print rejection sampling
98 + metrics.
99 + """
100 + if self._rank != 0:
101 + return False
102 +
103 + if (now - self._last_metrics_collect_time <
104 + self._rejsample_metrics_collect_interval_s):
105 + return False
106 + return True
107 +
108 + def _copy_rejsample_metrics_async(self) -> torch.cuda.Event:
109 + """Copy rejection sampling metrics (number of accepted tokens, etc) to
110 + CPU asynchronously.
111 +
112 + Returns a CUDA event recording when the copy is complete.
113 + """
114 + self._copy_stream.wait_stream(torch.cuda.current_stream())
115 +
116 + with torch.cuda.stream(self._copy_stream):
117 + self._aggregate_num_accepted_tokens.copy_(
118 + self._rejection_sampler.num_accepted_tokens, non_blocking=True)
119 + self._aggregate_num_emitted_tokens.copy_(
120 + self._rejection_sampler.num_emitted_tokens, non_blocking=True)
121 + # Number of draft tokens is calculated on CPU, so no copy is
122 + # required.
123 + self._aggregate_num_draft_tokens = (
124 + self._rejection_sampler.num_draft_tokens)
125 +
126 + aggregate_metrics_ready = torch.cuda.Event()
127 + aggregate_metrics_ready.record(self._copy_stream)
128 +
129 + return aggregate_metrics_ready
130 +
131 + def _collect_rejsample_metrics(
132 + self, k: int,
133 + ready_event: torch.cuda.Event) -> SpecDecodeWorkerMetrics:
134 + """Create metrics object from statistics copied asynchronously.
135 +
136 + Args:
137 + k: int. The number of speculative tokens; used to determine system
138 + efficiency.
139 + ready_event: torch.cuda.Event. The CUDA event recording when the
140 + async GPU->CPU copy is complete.
141 + """
142 +
143 + ready_event.synchronize()
144 + accepted_tokens = self._aggregate_num_accepted_tokens.item()
145 + emitted_tokens = self._aggregate_num_emitted_tokens.item()
146 + draft_tokens = self._aggregate_num_draft_tokens
147 +
148 + num_possible_tokens = self.get_max_num_accepted_tokens(draft_tokens, k)
149 +
150 + if draft_tokens > 0:
151 + draft_acceptance_rate = accepted_tokens / draft_tokens
152 + else:
153 + draft_acceptance_rate = float("nan")
154 +
155 + if num_possible_tokens > 0:
156 + system_efficiency = emitted_tokens / num_possible_tokens
157 + else:
158 + system_efficiency = float("nan")
159 +
160 + return SpecDecodeWorkerMetrics(
161 + num_spec_tokens=k,
162 + draft_acceptance_rate=draft_acceptance_rate,
163 + system_efficiency=system_efficiency,
164 + accepted_tokens=accepted_tokens,
165 + draft_tokens=draft_tokens,
166 + emitted_tokens=emitted_tokens,
167 + )
168 +
169 + @staticmethod
170 + def get_max_num_accepted_tokens(draft_tokens: int, k: int) -> int:
171 + # Divide by k since batch size can be variable.
172 + total_num_spec_seqs = draft_tokens / k
173 + num_accepted_per_seq_if_all_accepted = k + 1
174 + return int(total_num_spec_seqs / num_accepted_per_seq_if_all_accepted)
vllm/spec_decode/multi_step_worker.py new
+366
@@ -0,0 +1,366 @@
1 +from typing import List, Dict, Optional, Tuple
2 +import copy
3 +
4 +import torch
5 +
6 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 +from vllm.worker.worker import Worker
8 +from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeProposer
9 +from vllm.spec_decode.util import sampler_output_to_torch
10 +
11 +
12 +class MultiStepWorker(Worker):
13 + """The MultiStepWorker is equivalent to a Worker except that it allows
14 + multiple forward passes in a single call, assuming the scheduler has
15 + allocated enough space to store the additional KV. This reduces overhead
16 + by invoking the scheduler less.
17 +
18 + The MultiStepWorker does not support cache swap operations, or beam search.
19 + Cache swap operations do not require large modifications. On the other hand,
20 + beam search requires memory allocations during sequence forks and thus
21 + requires more thought for MultiStepWorker support.
22 + """
23 +
24 + def __init__(self, *args, **kwargs):
25 + super().__init__(*args, **kwargs)
26 +
27 + self._proposer: Optional[DraftModelTop1Proposer] = None
28 +
29 + def init_model(self):
30 + super().init_model()
31 +
32 + self._proposer = DraftModelTop1Proposer(
33 + self,
34 + self.device,
35 + self.max_model_len,
36 + self.vocab_size,
37 + )
38 +
39 + @torch.inference_mode()
40 + def execute_model_multi_step(
41 + self,
42 + seq_group_metadata_list: List[SequenceGroupMetadata],
43 + blocks_to_swap_in: Dict[int, int],
44 + blocks_to_swap_out: Dict[int, int],
45 + blocks_to_copy: Dict[int, List[int]],
46 + num_steps: int,
47 + ) -> List[SamplerOutput]:
48 + """Run the model forward pass num_steps times. Returns the list of
49 + sampler output, one per model forward pass.
50 + """
51 + self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
52 + blocks_to_swap_out, blocks_to_copy)
53 +
54 + # Shallow copy input data so modifications (such as appending tokens)
55 + # do not cause side-effects.
56 + copied_seq_group_metadata_list = self._shallow_copy_inputs(
57 + seq_group_metadata_list)
58 +
59 + # Assert enough KV space for num_steps tokens per sequence.
60 + self._assert_enough_kv_space(seq_group_metadata_list, num_steps)
61 +
62 + # Run model num_steps times.
63 + model_outputs = []
64 + for _ in range(num_steps):
65 + model_output = super().execute_model(
66 + seq_group_metadata_list=copied_seq_group_metadata_list,
67 + blocks_to_swap_in=blocks_to_swap_in,
68 + blocks_to_swap_out=blocks_to_swap_out,
69 + blocks_to_copy=blocks_to_copy,
70 + )
71 +
72 + self._append_new_tokens(model_output,
73 + copied_seq_group_metadata_list)
74 + model_outputs.append(model_output)
75 +
76 + return model_outputs
77 +
78 + def get_spec_proposals(
79 + self,
80 + seq_group_metadata_list: List[SequenceGroupMetadata],
81 + blocks_to_swap_in: Dict[int, int],
82 + blocks_to_swap_out: Dict[int, int],
83 + blocks_to_copy: Dict[int, List[int]],
84 + max_proposal_len: int,
85 + ) -> SpeculativeProposals:
86 + """Produce speculations given an input batch of sequences. The number of
87 + speculative tokens per sequence is determined by max_proposal_len.
88 + """
89 +
90 + return self._proposer.get_proposals(
91 + seq_group_metadata_list,
92 + blocks_to_swap_in,
93 + blocks_to_swap_out,
94 + blocks_to_copy,
95 + max_proposal_len,
96 + )
97 +
98 + def _append_new_tokens(
99 + self, model_output: SamplerOutput,
100 + seq_group_metadata_list: SequenceGroupMetadata) -> None:
101 + """Given model output from a single run, append the tokens to the
102 + sequences. This is normally done outside of the worker, but it is
103 + required if the worker is to perform multiple forward passes.
104 + """
105 + for seq_group_metadata, sequence_group_outputs in zip(
106 + seq_group_metadata_list, model_output):
107 + seq_group_metadata.is_prompt = False
108 +
109 + for seq_output in sequence_group_outputs.samples:
110 + # NOTE: Beam search is not supported, so we can assume that
111 + # parent_seq_id == seq_id.
112 + seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
113 +
114 + token_id = seq_output.output_token
115 + token_logprob = seq_output.logprobs[token_id]
116 +
117 + seq.append_token_id(token_id, token_logprob.logprob)
118 +
119 + def _shallow_copy_inputs(
120 + self, seq_group_metadata_list: List[SequenceGroupMetadata]
121 + ) -> List[SequenceGroupMetadata]:
122 + """Copy input data structures to remove side-effects when input data
123 + structures are shared with other modules.
124 +
125 + Helpful when the vLLM scheduler runs in the same process as the worker.
126 + The alternative is deep-copying (or other form of deep copy); this has
127 + performance downsides.
128 + """
129 +
130 + # Shallow-copy the list of SequenceGroupMetadata. This allows us to
131 + # append tokens and change is_prompt without external side-effects.
132 + new_seq_group_metadata_list = []
133 +
134 + for old_seq_group_metadata in seq_group_metadata_list:
135 + # We must shallow-copy seq_group_metadata as is_prompt could change.
136 + seq_group_metadata = copy.copy(old_seq_group_metadata)
137 + new_seq_group_metadata_list.append(seq_group_metadata)
138 +
139 + # We must shallow-copy seq_data as we will append token ids
140 + new_seq_data = {}
141 + for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
142 + new_seq_data[seq_id] = copy.copy(old_seq_data)
143 + new_seq_data[
144 + seq_id].output_token_ids = old_seq_data.output_token_ids[:]
145 +
146 + seq_group_metadata.seq_data = new_seq_data
147 +
148 + return new_seq_group_metadata_list
149 +
150 + def _assert_enough_kv_space(
151 + self, seq_group_metadata_list: List[SequenceGroupMetadata],
152 + num_steps: int) -> None:
153 + """Assert there are enough physical blocks per sequence to store the
154 + current KV plus additional KV from num_steps tokens.
155 + """
156 + assert self.model_runner.block_size is not None
157 + for seq_group_metadata in seq_group_metadata_list:
158 + # Only one seq_id is guaranteed because there is no beam search.
159 + seq_id = list(seq_group_metadata.seq_data.keys())[0]
160 + seq = seq_group_metadata.seq_data[seq_id]
161 +
162 + # After num_steps, the seq len will be the current seq len
163 + # plus one token per step.
164 + final_seq_len = seq.get_len() + num_steps
165 +
166 + # We will have final_seq_len - 1 KV because vLLM saves KV for a
167 + # token in the iteration after the token was generated.
168 + required_num_kv_slots = final_seq_len - 1
169 +
170 + # The allocated number of kv slots is the number of allocated blocks
171 + # times the number of slots of block.
172 + number_physical_blocks = len(
173 + seq_group_metadata.block_tables[seq_id])
174 + allocated_kv_slots = (number_physical_blocks *
175 + self.model_runner.block_size)
176 +
177 + if required_num_kv_slots > allocated_kv_slots:
178 + request_id = seq_group_metadata.request_id
179 + raise ValueError(
180 + "The worker attempted to run "
181 + f"{num_steps} times but found insufficient KV space for "
182 + f"{request_id=} {seq_id=}. ({allocated_kv_slots=} "
183 + f"{required_num_kv_slots=}).")
184 +
185 + def _raise_if_unsupported(
186 + self,
187 + seq_group_metadata_list: List[SequenceGroupMetadata],
188 + blocks_to_swap_in: Dict[int, int],
189 + blocks_to_swap_out: Dict[int, int],
190 + blocks_to_copy: Dict[int, List[int]],
191 + ) -> None:
192 + """MultiStepWorker does not yet implement support for cache swap
193 + operations or beam search.
194 + """
195 + if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
196 + raise NotImplementedError(
197 + "MultiStepWorker does not support cache operations")
198 +
199 + if any(
200 + len(seq_group_metadata.seq_data.keys()) != 1
201 + for seq_group_metadata in seq_group_metadata_list):
202 + raise NotImplementedError(
203 + "MultiStepWorker does not support beam search.")
204 +
205 +
206 +class DraftModelTop1Proposer(SpeculativeProposer):
207 + """Helper class which separates out sequences which would exceed the max
208 + model length when speculated upon.
209 +
210 + This allows combinations of models such as JackFram/llama-68m draft with
211 + meta-llama/Llama2-13b-chat-hf, as llama-68m has max_position_embeddings of
212 + 2048 while Llama2-13b has max_position_embeddings of 4096.
213 +
214 + We treat the sequences which exceed the proposal draft model length as
215 + "non-spec sequences". Essentially they skip the draft model and go through
216 + normal decoding in the target model.
217 +
218 + Currently, only proposal_lens of 0 and k are supported, where k is a global
219 + batch proposal length. In the future vLLM should support per-sequence
220 + proposal lengths.
221 + """
222 +
223 + def __init__(
224 + self,
225 + draft_worker: MultiStepWorker,
226 + device: str,
227 + max_model_len: int,
228 + vocab_size: int,
229 + ):
230 + self._draft_worker = draft_worker
231 + self._device = device
232 + self._max_model_len = max_model_len
233 + self._vocab_size = vocab_size
234 +
235 + def get_proposals(
236 + self,
237 + seq_group_metadata_list: List[SequenceGroupMetadata],
238 + blocks_to_swap_in: Dict[int, int],
239 + blocks_to_swap_out: Dict[int, int],
240 + blocks_to_copy: Dict[int, List[int]],
241 + max_proposal_len: int,
242 + ) -> SpeculativeProposals:
243 + """Get speculative proposals given the input batch.
244 +
245 + Sequences which would exceed the max model length are skipped during
246 + speculation.
247 + """
248 +
249 + # Split speculative- and non-speculative- sequences.
250 + proposal_lens, nonzero_proposal_len_seqs, nonzero_proposal_len_indices = self._split_by_max_model_len(
251 + seq_group_metadata_list, max_proposal_len)
252 +
253 + if nonzero_proposal_len_seqs:
254 + # Speculate tokens using the draft worker for the speculative
255 + # sequences.
256 + maybe_sampler_output = self._draft_worker.execute_model_multi_step(
257 + seq_group_metadata_list=nonzero_proposal_len_seqs,
258 + blocks_to_swap_in=blocks_to_swap_in,
259 + blocks_to_swap_out=blocks_to_swap_out,
260 + blocks_to_copy=blocks_to_copy,
261 + num_steps=max_proposal_len,
262 + )
263 + else:
264 + # If no sequences can be speculated, set sampler output to None.
265 + maybe_sampler_output = None
266 +
267 + # Combine speculative- and non-speculative sequences into the same
268 + # representation.
269 + proposal_tokens, proposal_probs, proposal_lens = self._merge_outputs(
270 + batch_size=len(seq_group_metadata_list),
271 + max_proposal_len=max_proposal_len,
272 + maybe_sampler_output=maybe_sampler_output,
273 + proposal_lens=proposal_lens,
274 + nonzero_proposal_len_indices=nonzero_proposal_len_indices,
275 + )
276 +
277 + proposals = SpeculativeProposals(
278 + proposal_token_ids=proposal_tokens,
279 + proposal_probs=proposal_probs,
280 + proposal_lens=proposal_lens,
281 + )
282 +
283 + return proposals
284 +
285 + def _split_by_max_model_len(
286 + self,
287 + seq_group_metadata_list: List[SequenceGroupMetadata],
288 + max_proposal_len: int,
289 + ) -> Tuple[List[int], List[SequenceGroupMetadata], List[int]]:
290 + """Determine which sequences would exceed the max model length.
291 + """
292 +
293 + proposal_lens: List[int] = []
294 + nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
295 + nonzero_proposal_len_indices: List[int] = []
296 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
297 + seq_data = next(iter(seq_group_metadata.seq_data.values()))
298 + seq_len = seq_data.get_len()
299 +
300 + # Currently only proposal lens of 0 or the global batch proposal len
301 + # are supported.
302 + if seq_len + max_proposal_len < self._max_model_len:
303 + proposal_lens.append(max_proposal_len)
304 + nonzero_proposal_len_seqs.append(seq_group_metadata)
305 + nonzero_proposal_len_indices.append(i)
306 + else:
307 + proposal_lens.append(0)
308 +
309 + return proposal_lens, nonzero_proposal_len_seqs, nonzero_proposal_len_indices
310 +
311 + def _merge_outputs(
312 + self,
313 + batch_size: int,
314 + max_proposal_len: int,
315 + maybe_sampler_output: Optional[SamplerOutput],
316 + proposal_lens: List[int],
317 + nonzero_proposal_len_indices: List[int],
318 + ) -> Tuple[torch.Tensor, torch.tensor, torch.Tensor]:
319 + """After speculations are produced, merge the speculation results with
320 + the skipped sequences.
321 + """
322 + if maybe_sampler_output is None:
323 + # If no speculative tokens, the sampler output will be None.
324 + # In this case we return empty tensors.
325 + proposal_tokens = torch.zeros(0,
326 + max_proposal_len,
327 + dtype=torch.long,
328 + device=self._device)
329 + proposal_probs = torch.zeros(0,
330 + max_proposal_len,
331 + self._vocab_size,
332 + dtype=torch.float32,
333 + device=self._device)
334 + proposal_lens = torch.zeros(len(proposal_lens),
335 + dtype=torch.long,
336 + device=self._device)
337 + return proposal_tokens, proposal_probs, proposal_lens
338 +
339 + sampler_output = maybe_sampler_output
340 +
341 + proposal_tokens, proposal_probs = sampler_output_to_torch(
342 + sampler_output)
343 +
344 + # Now, reformat the output GPU tensors such that each sequence has
345 + # a proposal. the proposal can be empty, e.g. [-1, -1, -1]
346 +
347 + entire_proposal_tokens = torch.full(size=(batch_size,
348 + *proposal_tokens.shape[1:]),
349 + fill_value=-1,
350 + dtype=torch.long,
351 + device=self._device)
352 + entire_proposal_tokens[nonzero_proposal_len_indices] = proposal_tokens
353 + entire_proposal_probs = torch.zeros(batch_size,
354 + *proposal_probs.shape[1:],
355 + dtype=torch.float32,
356 + device=self._device)
357 + entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
358 +
359 + proposal_tokens, proposal_probs = entire_proposal_tokens, entire_proposal_probs
360 +
361 + proposal_lens = torch.zeros(batch_size,
362 + dtype=torch.long,
363 + device=self._device)
364 + proposal_lens[nonzero_proposal_len_indices] = max_proposal_len
365 +
366 + return proposal_tokens, proposal_probs, proposal_lens
vllm/spec_decode/spec_decode_worker.py new
+372
@@ -0,0 +1,372 @@
1 +from typing import List, Tuple, Optional, Dict
2 +from functools import cached_property
3 +
4 +import torch
5 +
6 +from vllm.spec_decode.metrics import AsyncMetricsCollector
7 +from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
8 + SequenceGroupOutput, SequenceOutput)
9 +from vllm.worker.worker import Worker
10 +from vllm.spec_decode.multi_step_worker import MultiStepWorker
11 +from vllm.model_executor.layers.rejection_sampler import RejectionSampler
12 +from vllm.config import CacheConfig
13 +from vllm.spec_decode.util import nvtx_range, get_all_seq_ids, split_batch_by_proposal_len
14 +from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeScores
15 +from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
16 +from vllm.spec_decode.interfaces import SpeculativeScorer
17 +
18 +
19 +class SpecDecodeWorker:
20 + """Worker which implements speculative decoding.
21 +
22 + Speculative decoding reduces decoding per-token latency by using a proposal
23 + method, such as a small draft model, to speculate ahead of a larger LLM. The
24 + probabilities of the speculative tokens are then determined by the larger
25 + LLM, after which some verification routine determines which (if any) of the
26 + speculative tokens are accepted by the larger LLM.
27 +
28 + See https://github.com/vllm-project/vllm/pull/2188 and
29 + https://github.com/vllm-project/vllm/pull/3103 for more info.
30 +
31 + The current implementation has the following limitations:
32 + * Only draft-model proposal is implemented (contributions for more forms are
33 + welcome!).
34 + * Only top-1 proposal and scoring are implemented. Tree-attention is left as
35 + future work.
36 + * Only lossless rejection sampling is supported. Contributions adding lossy
37 + verification routines are welcome (e.g. Medusa's typical acceptance).
38 + * All sequences in a batch must have the same proposal length, or zero. This
39 + can be improved by having per-sequence speculation in the future.
40 + * The scoring forward pass is done without an MQA kernel, which is
41 + suboptimal especially as the batch size, proposal length, and sequence
42 + lengths grow. Contributions to add a MQA scoring are welcome once
43 + correctness tests pass.
44 + More info here https://docs.google.com/document/d/1T-JaS2T1NRfdP51qzqpyakoCXxSXTtORppiwaj5asxA/edit.
45 + """
46 +
47 + def __init__(
48 + self,
49 + proposer_worker: MultiStepWorker,
50 + scorer_worker: Worker,
51 + rejection_sampler: RejectionSampler,
52 + metrics_collector: Optional[AsyncMetricsCollector] = None,
53 + ):
54 + """
55 + Create a SpecDecodeWorker.
56 +
57 + Args:
58 + proposer_worker: A worker that can produce speculative tokens for
59 + sequences.
60 + scorer_worker: A worker that produces probabilities of speculative
61 + tokens according to some base model. Typically a vanilla vLLM
62 + Worker.
63 + rejection_sampler: A Torch module used to perform modified rejection
64 + sampling for speculative decoding.
65 + metrics_collector: Helper class for collecting metrics; can be set
66 + for testing purposes.
67 + """
68 + self.proposer_worker = proposer_worker
69 + self.scorer_worker = scorer_worker
70 + self.rejection_sampler = rejection_sampler
71 +
72 + self._metrics = AsyncMetricsCollector(
73 + rejection_sampler
74 + ) if metrics_collector is None else metrics_collector
75 +
76 + self.probs_dtype = self.rejection_sampler.probs_dtype
77 + self.token_id_dtype = self.rejection_sampler.token_id_dtype
78 +
79 + self.scorer: SpeculativeScorer = None
80 +
81 + def init_model(self) -> None:
82 + """Initialize both scorer and proposer models.
83 + """
84 + # The scorer worker model is initialized first in case the proposer
85 + # model has a smaller TP degree than the target worker.
86 + self.scorer_worker.init_model()
87 + self.proposer_worker.init_model()
88 +
89 + self._metrics.init_gpu_tensors(self.rank)
90 + self.rejection_sampler.init_gpu_tensors(self.rank)
91 + self.scorer = BatchExpansionTop1Scorer(
92 + scorer_worker=self.scorer_worker,
93 + device=self.device,
94 + vocab_size=self._vocab_size)
95 +
96 + def profile_num_available_blocks(self, block_size: int,
97 + gpu_memory_utilization: float,
98 + cpu_swap_space: int,
99 + cache_dtype: str) -> Tuple[int, int]:
100 + """Determine the number of cache blocks to use.
101 +
102 + This is done by profiling the scorer model (which is typically the
103 + larger of the two). Then the total memory which would be used by the
104 + scorer cache is divided evenly between the proposer and scorer model KV,
105 + such that the number of blocks is equal in both KV caches.
106 + """
107 + num_gpu_blocks, num_cpu_blocks = (
108 + self.scorer_worker.profile_num_available_blocks(
109 + block_size, gpu_memory_utilization, cpu_swap_space,
110 + cache_dtype))
111 +
112 + scorer_cache_block_size_bytes = self.scorer_worker.get_cache_block_size_bytes(
113 + block_size, cache_dtype)
114 + proposer_cache_block_size_bytes = self.proposer_worker.get_cache_block_size_bytes(
115 + block_size, cache_dtype)
116 +
117 + new_num_gpu_blocks = split_num_cache_blocks_evenly(
118 + scorer_cache_block_size_bytes, proposer_cache_block_size_bytes,
119 + num_gpu_blocks)
120 + return new_num_gpu_blocks, num_cpu_blocks
121 +
122 + def init_cache_engine(self, cache_config: CacheConfig):
123 + """Initialize the cache engine of the scorer and proposer workers.
124 + """
125 + self.scorer_worker.init_cache_engine(cache_config)
126 + self.proposer_worker.init_cache_engine(cache_config)
127 +
128 + @torch.inference_mode()
129 + def execute_model(
130 + self,
131 + seq_group_metadata_list: List[SequenceGroupMetadata],
132 + blocks_to_swap_in: Optional[Dict[int, int]],
133 + blocks_to_swap_out: Optional[Dict[int, int]],
134 + blocks_to_copy: Optional[Dict[int, List[int]]],
135 + num_spec_tokens: int,
136 + ) -> List[SamplerOutput]:
137 + """Perform speculative decoding on the input batch.
138 + """
139 +
140 + assert seq_group_metadata_list is not None, (
141 + "speculative decoding "
142 + "requires non-None seq_group_metadata_list")
143 +
144 + # If no spec tokens, call the proposer and scorer workers normally.
145 + # Used for prefill.
146 + if num_spec_tokens == 0 or len(seq_group_metadata_list) == 0:
147 + return self._run_no_spec(
148 + seq_group_metadata_list=seq_group_metadata_list,
149 + blocks_to_swap_in=blocks_to_swap_in,
150 + blocks_to_swap_out=blocks_to_swap_out,
151 + blocks_to_copy=blocks_to_copy,
152 + )
153 +
154 + return self._run_speculative_decoding_step(
155 + seq_group_metadata_list=seq_group_metadata_list,
156 + blocks_to_swap_in=blocks_to_swap_in,
157 + blocks_to_swap_out=blocks_to_swap_out,
158 + blocks_to_copy=blocks_to_copy,
159 + k=num_spec_tokens,
160 + )
161 +
162 + @nvtx_range("spec_decode_worker._run_no_spec")
163 + def _run_no_spec(
164 + self,
165 + seq_group_metadata_list: List[SequenceGroupMetadata],
166 + blocks_to_swap_in: Optional[Dict[int, int]],
167 + blocks_to_swap_out: Optional[Dict[int, int]],
168 + blocks_to_copy: Optional[Dict[int, List[int]]],
169 + ) -> List[SamplerOutput]:
170 + """Run a prefill step, without any speculation. The input is sent to the
171 + proposer and scorer model so that the KV cache is consistent between the
172 + two.
173 + """
174 +
175 + self.proposer_worker.execute_model(
176 + seq_group_metadata_list=seq_group_metadata_list,
177 + blocks_to_swap_in=blocks_to_swap_in,
178 + blocks_to_swap_out=blocks_to_swap_out,
179 + blocks_to_copy=blocks_to_copy,
180 + return_python_output=False)
181 +
182 + sampler_output = self.scorer_worker.execute_model(
183 + seq_group_metadata_list=seq_group_metadata_list,
184 + blocks_to_swap_in=blocks_to_swap_in,
185 + blocks_to_swap_out=blocks_to_swap_out,
186 + blocks_to_copy=blocks_to_copy,
187 + )
188 +
189 + # Clear device tensors from sampler output. This reduces communication
190 + # overhead when the engine runs in a different process than the workers.
191 + sampler_output.probs = None
192 + sampler_output.sampled_tokens = None
193 + return [sampler_output]
194 +
195 + @nvtx_range("spec_decode_worker._run_speculative_decoding_step")
196 + def _run_speculative_decoding_step(
197 + self,
198 + seq_group_metadata_list: List[SequenceGroupMetadata],
199 + blocks_to_swap_in: Optional[Dict[int, int]],
200 + blocks_to_swap_out: Optional[Dict[int, int]],
201 + blocks_to_copy: Optional[Dict[int, List[int]]],
202 + k: int,
203 + ) -> List[SamplerOutput]:
204 + """Execute a single step of speculative decoding.
205 +
206 + This invokes the proposer worker to get k speculative tokens for each
207 + sequence, then scores each speculative token using the scoring worker.
208 +
209 + Returns a list of SamplerOutput, each containing a single token per
210 + sequence.
211 + """
212 +
213 + # Generate proposals using draft worker.
214 + proposals = self.proposer_worker.get_spec_proposals(
215 + seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
216 + blocks_to_copy, k)
217 +
218 + proposal_scores = self.scorer.score_proposals(
219 + seq_group_metadata_list,
220 + blocks_to_swap_in,
221 + blocks_to_swap_out,
222 + blocks_to_copy,
223 + k,
224 + proposals,
225 + )
226 +
227 + accepted_token_ids = self._verify_tokens(seq_group_metadata_list,
228 + proposal_scores, proposals, k)
229 +
230 + return self._create_output_sampler_list(seq_group_metadata_list,
231 + accepted_token_ids, k)
232 +
233 + @nvtx_range("spec_decode_worker._verify_tokens")
234 + def _verify_tokens(
235 + self,
236 + seq_group_metadata_list: List[SequenceGroupMetadata],
237 + proposal_scores: SpeculativeScores,
238 + proposals: SpeculativeProposals,
239 + max_proposal_len: int,
240 + ) -> torch.Tensor:
241 + """Determine which speculative tokens are accepted using the
242 + probabilities of each token according to the proposer and scorer models.
243 + """
244 + proposal_lens_list = proposals.proposal_lens.tolist()
245 +
246 + # vLLM currently only supports proposal lens equal to zero or the batch
247 + # proposal len. This adds some complexity (splitting the batch into spec
248 + # and non spec sequences) and should be removed in the future. It can be
249 + # done by supporting per-sequence proposal lens.
250 + _, spec_indices = split_batch_by_proposal_len(
251 + seq_group_metadata_list,
252 + proposal_lens_list,
253 + select_proposal_len_zero=False)
254 + _, non_spec_indices = split_batch_by_proposal_len(
255 + seq_group_metadata_list,
256 + proposal_lens_list,
257 + select_proposal_len_zero=True)
258 + original_indices = spec_indices + non_spec_indices
259 +
260 + proposal_probs = proposal_scores.probs[spec_indices, :-1]
261 + bonus_token_ids = proposal_scores.token_ids[spec_indices, -1:]
262 + non_spec_token_ids = proposal_scores.token_ids[non_spec_indices]
263 +
264 + accepted_token_ids = self.rejection_sampler(
265 + proposal_probs,
266 + bonus_token_ids,
267 + proposals.proposal_probs,
268 + proposals.proposal_token_ids,
269 + )
270 +
271 + # Append output tokens from non-speculative sequences to
272 + # the accepted token ids tensor.
273 + non_spec_token_ids = non_spec_token_ids.expand(-1, max_proposal_len +
274 + 1).clone()
275 + non_spec_token_ids[:, 1:] = -1
276 + accepted_token_ids = torch.cat(
277 + [accepted_token_ids, non_spec_token_ids])
278 +
279 + # Rearrange so that results are in the order of the original seq group
280 + # metadata.
281 + accepted_token_ids[original_indices] = accepted_token_ids.clone()
282 +
283 + return accepted_token_ids
284 +
285 + def _create_output_sampler_list(
286 + self,
287 + seq_group_metadata_list: List[SequenceGroupMetadata],
288 + accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1]
289 + k: int,
290 + ) -> List[SamplerOutput]:
291 + """Given the accepted token ids, create a list of SamplerOutput.
292 +
293 + The output is padded with -1 tokens such that each sequence has
294 + the same number of outputs.
295 + """
296 + seq_ids = get_all_seq_ids(seq_group_metadata_list)
297 +
298 + # shape: [k+1, batch_size]
299 + accepted_token_ids_by_step = accepted_token_ids.transpose(0,
300 + 1).tolist()
301 + sampler_output_list = []
302 + for token_ids_by_step in accepted_token_ids_by_step:
303 + if all(token_id == -1 for token_id in token_ids_by_step):
304 + break
305 +
306 + step_output_token_ids = []
307 + for token_id, seq_id in zip(token_ids_by_step, seq_ids):
308 + step_output_token_ids.append(
309 + SequenceGroupOutput(
310 + samples=[
311 + SequenceOutput(
312 + parent_seq_id=seq_id,
313 + output_token=token_id,
314 + # TODO Add verifier logprobs.
315 + logprobs={token_id: 0.0},
316 + )
317 + ],
318 + prompt_logprobs=None,
319 + ))
320 + sampler_output_list.append(
321 + SamplerOutput(outputs=step_output_token_ids))
322 +
323 + maybe_rejsample_metrics = self._metrics.maybe_collect_rejsample_metrics(
324 + k)
325 + if maybe_rejsample_metrics is not None:
326 + sampler_output_list[
327 + 0].spec_decode_worker_metrics = maybe_rejsample_metrics
328 +
329 + return sampler_output_list
330 +
331 + @cached_property
332 + def _vocab_size(self) -> int:
333 + """Get the vocab size of the model and make sure it's consistent between
334 + draft and target workers.
335 + """
336 + vocab_sizes = [
337 + worker.vocab_size
338 + for worker in [self.proposer_worker, self.scorer_worker]
339 + ]
340 + assert all(vocab_sizes[0] == vocab_size for vocab_size in vocab_sizes)
341 + return vocab_sizes[0]
342 +
343 + @property
344 + def rank(self):
345 + return self.scorer_worker.rank
346 +
347 + @property
348 + def device(self):
349 + return self.scorer_worker.device
350 +
351 +
352 +def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
353 + proposer_cache_block_size_bytes: int,
354 + total_num_gpu_blocks: int) -> int:
355 + """Given total_num_gpu_blocks, the number of GPU blocks that could be
356 + allocate to the target model, this function calculates how many blocks
357 + should be given to the draft and target model.
358 +
359 + Note that usually the block size, in bytes, of each model is different,
360 + as it's a function of number of KV/layer, number of heads, and hidden
361 + dimension size.
362 +
363 + Since the target and draft models allocate the same number of blocks, we
364 + simply calculate the number of blocks where if allocated by both models,
365 + the total memory usage from KV cache is no larger than the number of
366 + blocks allocatable by the target model alone.
367 + """
368 + new_num_gpu_blocks = int(
369 + total_num_gpu_blocks * scorer_cache_block_size_bytes /
370 + (proposer_cache_block_size_bytes + scorer_cache_block_size_bytes))
371 +
372 + return new_num_gpu_blocks
vllm/spec_decode/util.py new
+99
@@ -0,0 +1,99 @@
1 +import torch
2 +from typing import List, Tuple
3 +from vllm.sequence import SequenceGroupMetadata, SamplerOutput
4 +from contextlib import contextmanager
5 +from itertools import chain
6 +
7 +SeqId = int
8 +
9 +
10 +def get_all_seq_ids(
11 + seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[SeqId]:
12 + """Given a list of SequenceGroupMetadata, create a list of all
13 + sequence ids.
14 + """
15 + return list(
16 + chain.from_iterable([
17 + seq_group_metadata.seq_data.keys()
18 + for seq_group_metadata in seq_group_metadata_list
19 + ]))
20 +
21 +
22 +def split_batch_by_proposal_len(
23 + seq_group_metadata_list: List[SequenceGroupMetadata],
24 + proposal_lens: List[int], select_proposal_len_zero: bool
25 +) -> Tuple[List[SequenceGroupMetadata], List[int]]:
26 + """Utility function that splits a batch based on whether the proposal len is
27 + zero or not. We should remove this once vLLM supports per-sequence proposal
28 + lens in a batch.
29 + """
30 +
31 + if select_proposal_len_zero:
32 + predicate = lambda proposal_len: proposal_len == 0
33 + else:
34 + predicate = lambda proposal_len: proposal_len != 0
35 +
36 + indices = [
37 + i for i, (_, proposal_len
38 + ) in enumerate(zip(seq_group_metadata_list, proposal_lens))
39 + if predicate(proposal_len)
40 + ]
41 + seq_groups = [
42 + seq_group for seq_group, proposal_len in zip(
43 + seq_group_metadata_list, proposal_lens) if predicate(proposal_len)
44 + ]
45 +
46 + return seq_groups, indices
47 +
48 +
49 +def sampler_output_to_torch(
50 + sampler_output_list: List[SamplerOutput],
51 +) -> Tuple[torch.Tensor, torch.Tensor]:
52 + """Utility function which converts a list of SamplerOutput to tensors.
53 +
54 + Returns:
55 + sampled_token_ids: torch.Tensor
56 + shape: [batch_size, len(sampler_output_list)]
57 +
58 + sampled_token_probs: torch.Tensor
59 + shape: [batch_size, len(sampler_output_list), vocab_size]
60 + """
61 +
62 + # shape: [batch_size, num_sampler_output, vocab_size]
63 + sampled_token_probs = torch.stack(
64 + [
65 + sampler_output.sampled_token_probs
66 + for sampler_output in sampler_output_list
67 + ],
68 + dim=0,
69 + ).transpose(0, 1)
70 +
71 + # shape: [batch_size, num_sampler_output]
72 + sampled_token_ids = torch.stack(
73 + [
74 + sampler_output.sampled_token_ids.flatten()
75 + for sampler_output in sampler_output_list
76 + ],
77 + dim=0,
78 + ).transpose(0, 1)
79 +
80 + return sampled_token_ids, sampled_token_probs
81 +
82 +
83 +@contextmanager
84 +def nvtx_range(msg, *args, **kwargs):
85 + """
86 + Context manager / decorator that pushes an NVTX range at the beginning
87 + of its scope, and pops it at the end. If extra arguments are given,
88 + they are passed as arguments to msg.format().
89 +
90 + If running with cuda graphs, you must enable nsys cuda graph profiling.
91 +
92 + Arguments:
93 + msg (string): message to associate with the range
94 + """
95 + torch.cuda.nvtx.range_push(msg.format(*args, **kwargs))
96 + try:
97 + yield
98 + finally:
99 + torch.cuda.nvtx.range_pop()
vllm/worker/model_runner.py
+6 -5
@@ -97,8 +97,6 @@ class ModelRunner:
97 f"Loading model weights took {self.model_memory_usage / float(2**30):.4f} GB"
98 )
99
100 - vocab_size = self.model.config.vocab_size
101 -
100 if self.lora_config:
101 assert hasattr(
102 self.model, "supported_lora_modules"
@@ -111,7 +109,7 @@ class ModelRunner:
109 self.lora_manager = LRUCacheWorkerLoRAManager(
110 self.scheduler_config.max_num_seqs,
111 self.scheduler_config.max_num_batched_tokens +
114 - self.scheduler_config.max_paddings, vocab_size,
112 + self.scheduler_config.max_paddings, self.vocab_size,
113 self.lora_config, self.device, self.model.embedding_modules,
114 self.model.embedding_padding_modules)
115 self.model = self.lora_manager.create_lora_manager(self.model)
@@ -607,8 +605,7 @@ class ModelRunner:
605 @torch.inference_mode()
606 def profile_run(self) -> None:
607 # Enable top-k sampling to reflect the accurate memory usage.
610 - vocab_size = self.model_config.get_vocab_size()
611 - sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
608 + sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
609 max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
610 max_num_seqs = self.scheduler_config.max_num_seqs
611
@@ -774,6 +771,10 @@ class ModelRunner:
771 self.graph_runners.clear()
772 self.cupy_nccl_backend = None
773
774 + @property
775 + def vocab_size(self) -> int:
776 + return self.model_config.get_vocab_size()
777 +
778
779 class CUDAGraphRunner:
780
vllm/worker/spec_decode/multi_step_worker.py deleted
-178
@@ -1,178 +0,0 @@
1 -from typing import List, Dict
2 -import copy
3 -
4 -import torch
5 -
6 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 -from vllm.worker.worker import Worker
8 -
9 -
10 -class MultiStepWorker(Worker):
11 - """The MultiStepWorker is equivalent to a Worker except that it allows
12 - multiple forward passes in a single call, assuming the scheduler has
13 - allocated enough space to store the additional KV. This reduces overhead
14 - by invoking the scheduler less.
15 -
16 - The MultiStepWorker does not support cache swap operations, or beam search.
17 - Cache swap operations do not require large modifications. On the other hand,
18 - beam search requires memory allocations during sequence forks and thus
19 - requires more thought for MultiStepWorker support.
20 - """
21 -
22 - @torch.inference_mode()
23 - def execute_model_multi_step(
24 - self,
25 - seq_group_metadata_list: List[SequenceGroupMetadata],
26 - blocks_to_swap_in: Dict[int, int],
27 - blocks_to_swap_out: Dict[int, int],
28 - blocks_to_copy: Dict[int, List[int]],
29 - num_steps: int,
30 - ) -> List[SamplerOutput]:
31 - """Run the model forward pass num_steps times. Returns the list of
32 - sampler output, one per model forward pass.
33 - """
34 - self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
35 - blocks_to_swap_out, blocks_to_copy)
36 -
37 - # Shallow copy input data so modifications (such as appending tokens)
38 - # do not cause side-effects.
39 - copied_seq_group_metadata_list = self._shallow_copy_inputs(
40 - seq_group_metadata_list)
41 -
42 - # Assert enough KV space for num_steps tokens per sequence.
43 - self._assert_enough_kv_space(seq_group_metadata_list, num_steps)
44 -
45 - # Run model num_steps times.
46 - model_outputs = []
47 - for _ in range(num_steps):
48 - model_output = super().execute_model(
49 - seq_group_metadata_list=copied_seq_group_metadata_list,
50 - blocks_to_swap_in=blocks_to_swap_in,
51 - blocks_to_swap_out=blocks_to_swap_out,
52 - blocks_to_copy=blocks_to_copy,
53 - )
54 -
55 - self._append_new_tokens(model_output,
56 - copied_seq_group_metadata_list)
57 - model_outputs.append(model_output)
58 -
59 - return model_outputs
60 -
61 - def _append_new_tokens(
62 - self, model_output: SamplerOutput,
63 - seq_group_metadata_list: SequenceGroupMetadata) -> None:
64 - """Given model output from a single run, append the tokens to the
65 - sequences. This is normally done outside of the worker, but it is
66 - required if the worker is to perform multiple forward passes.
67 - """
68 - for seq_group_metadata, sequence_group_outputs in zip(
69 - seq_group_metadata_list, model_output):
70 - seq_group_metadata.is_prompt = False
71 -
72 - for seq_output in sequence_group_outputs.samples:
73 - # NOTE: Beam search is not supported, so we can assume that
74 - # parent_seq_id == seq_id.
75 - seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
76 -
77 - token_id = seq_output.output_token
78 - token_logprob = seq_output.logprobs[token_id]
79 -
80 - seq.append_token_id(token_id, token_logprob.logprob)
81 -
82 - def _shallow_copy_inputs(
83 - self, seq_group_metadata_list: List[SequenceGroupMetadata]
84 - ) -> List[SequenceGroupMetadata]:
85 - """Copy input data structures to remove side-effects when input data
86 - structures are shared with other modules.
87 -
88 - The multi-step worker must be able to append tokens to sequences after
89 - a forward pass. This necessitates modification of the data structures
90 - used by the worker. Since these data structures are shared with other
91 - parts of vLLM, like the scheduler, we must take care not to introduce
92 - unexpected side-effects.
93 -
94 - When Ray is used to orchestrate worker processes (such as when the
95 - tensor-parallel degree is >1), this is not a problem because the input
96 - datastructures will be serialized and created anew in the worker
97 - process.
98 -
99 - However, when Ray is not used to orchestrate the worker processes (such
100 - as when the tensor-parallel degree is 1), this is a problem. We avoid
101 - the problem by shallow-copying the input datastructures (specifically,
102 - the parts that will change in multiple steps).
103 - """
104 -
105 - # Shallow-copy the list of SequenceGroupMetadata. This allows us to
106 - # append tokens and change is_prompt without external side-effects.
107 - new_seq_group_metadata_list = []
108 -
109 - for old_seq_group_metadata in seq_group_metadata_list:
110 - # We must shallow-copy seq_group_metadata as is_prompt could change.
111 - seq_group_metadata = copy.copy(old_seq_group_metadata)
112 - new_seq_group_metadata_list.append(seq_group_metadata)
113 -
114 - # We must shallow-copy seq_data as we will append token ids
115 - new_seq_data = {}
116 - for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
117 - new_seq_data[seq_id] = copy.copy(old_seq_data)
118 - new_seq_data[
119 - seq_id].output_token_ids = old_seq_data.output_token_ids[:]
120 -
121 - seq_group_metadata.seq_data = new_seq_data
122 -
123 - return new_seq_group_metadata_list
124 -
125 - def _assert_enough_kv_space(
126 - self, seq_group_metadata_list: List[SequenceGroupMetadata],
127 - num_steps: int) -> None:
128 - """Assert there are enough physical blocks per sequence to store the
129 - current KV plus additional KV from num_steps tokens.
130 - """
131 - assert self.model_runner.block_size is not None
132 - for seq_group_metadata in seq_group_metadata_list:
133 - # Only one seq_id is guaranteed because there is no beam search.
134 - seq_id = list(seq_group_metadata.seq_data.keys())[0]
135 - seq = seq_group_metadata.seq_data[seq_id]
136 -
137 - # After num_steps, the seq len will be the current seq len
138 - # plus one token per step.
139 - final_seq_len = seq.get_len() + num_steps
140 -
141 - # We will have final_seq_len - 1 KV because vLLM saves KV for a
142 - # token in the iteration after the token was generated.
143 - required_num_kv_slots = final_seq_len - 1
144 -
145 - # The allocated number of kv slots is the number of allocated blocks
146 - # times the number of slots of block.
147 - number_physical_blocks = len(
148 - seq_group_metadata.block_tables[seq_id])
149 - allocated_kv_slots = (number_physical_blocks *
150 - self.model_runner.block_size)
151 -
152 - if required_num_kv_slots > allocated_kv_slots:
153 - request_id = seq_group_metadata.request_id
154 - raise ValueError(
155 - "The worker attempted to run "
156 - f"{num_steps} times but found insufficient KV space for "
157 - f"{request_id=} {seq_id=}. ({allocated_kv_slots=} "
158 - f"{required_num_kv_slots=}).")
159 -
160 - def _raise_if_unsupported(
161 - self,
162 - seq_group_metadata_list: List[SequenceGroupMetadata],
163 - blocks_to_swap_in: Dict[int, int],
164 - blocks_to_swap_out: Dict[int, int],
165 - blocks_to_copy: Dict[int, List[int]],
166 - ) -> None:
167 - """MultiStepWorker does not yet implement support for cache swap
168 - operations or beam search.
169 - """
170 - if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
171 - raise NotImplementedError(
172 - "MultiStepWorker does not support cache operations")
173 -
174 - if any(
175 - len(seq_group_metadata.seq_data.keys()) != 1
176 - for seq_group_metadata in seq_group_metadata_list):
177 - raise NotImplementedError(
178 - "MultiStepWorker does not support beam search.")
vllm/worker/worker.py
+18 -2
@@ -130,8 +130,8 @@ class Worker:
130 # GPU did not change their memory usage during the profiling.
131 peak_memory = self.init_gpu_memory - free_gpu_memory
132
133 - cache_block_size = CacheEngine.get_cache_block_size(
134 - block_size, cache_dtype, self.model_config, self.parallel_config)
133 + cache_block_size = self.get_cache_block_size_bytes(
134 + block_size, cache_dtype)
135 num_gpu_blocks = int(
136 (total_gpu_memory * gpu_memory_utilization - peak_memory) //
137 cache_block_size)
@@ -232,6 +232,22 @@ class Worker:
232 def list_loras(self) -> Set[int]:
233 return self.model_runner.list_loras()
234
235 + @property
236 + def max_model_len(self) -> int:
237 + return self.model_config.max_model_len
238 +
239 + @property
240 + def vocab_size(self) -> int:
241 + return self.model_runner.vocab_size
242 +
243 + def get_cache_block_size_bytes(self, block_size: int,
244 + cache_dtype: str) -> int:
245 + """Get the size of the KV cache block size in bytes.
246 + """
247 + return CacheEngine.get_cache_block_size(block_size, cache_dtype,
248 + self.model_config,
249 + self.parallel_config)
250 +
251
252 def init_distributed_environment(
253 parallel_config: ParallelConfig,