A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Speculative decoding 2/9] Multi-step worker for draft model (#2424)
Cade Daniel committed
Jan 21, 2024 at 16:31 UTC
18bfcdd05c657e6997b132488e6f4e74307d6cee
11 files changed
+658
-12
tests/worker/__init__.py
tests/worker/spec_decode/__init__.py
tests/worker/spec_decode/test_multi_step_worker.py
new
+261
@@ -0,0 +1,261 @@
1
+import torch
2
+import random
3
+import pytest
4
+from unittest.mock import MagicMock
5
+
6
+from vllm.worker.spec_decode.multi_step_worker import MultiStepWorker
7
+from vllm.worker.worker import Worker
8
+from vllm.model_executor.utils import set_random_seed
9
+
10
+from .utils import (create_execute_model_data, create_worker,
11
+ create_seq_group_metadata_from_prompts, zero_kv_cache,
12
+ patch_execute_model_with_seeds,
13
+ assert_logprobs_dict_allclose)
14
+
15
+
16
+@pytest.mark.parametrize('num_steps', list(range(1, 17)))
17
+def test_assert_enough_kv_space(num_steps: int):
18
+ """Test that the multi step worker checks for sufficient space in the KV
19
+ cache. It should throw if it cannot run all the steps.
20
+ """
21
+ block_size = 16
22
+ num_gpu_blocks = 2048 // block_size
23
+
24
+ prompts = [
25
+ list(range(block_size * 3)),
26
+ list(range(block_size * 2)),
27
+ ]
28
+
29
+ prev_output_tokens = [
30
+ list(range(block_size * 1)),
31
+ list(range(block_size * 2)),
32
+ ]
33
+
34
+ final_seq_lens = [
35
+ len(prompt + output) + num_steps
36
+ for prompt, output in zip(prompts, prev_output_tokens)
37
+ ]
38
+
39
+ inputs = create_seq_group_metadata_from_prompts(
40
+ prompts,
41
+ num_gpu_blocks,
42
+ block_size,
43
+ final_seq_lens,
44
+ continuations=prev_output_tokens)
45
+
46
+ assert_enough_kv_space = MultiStepWorker._assert_enough_kv_space # pylint: disable=protected-access
47
+ worker = MagicMock()
48
+ worker.model_runner.block_size = block_size
49
+
50
+ for seq_group_metadata in inputs:
51
+ original_block_tables = seq_group_metadata.block_tables
52
+
53
+ # No exception.
54
+ assert_enough_kv_space(worker, inputs, num_steps)
55
+
56
+ seq_group_metadata.block_tables = {
57
+ seq_id: []
58
+ for seq_id, physical_blocks in original_block_tables.items()
59
+ }
60
+
61
+ # Expect exception.
62
+ with pytest.raises(ValueError,
63
+ match='times but found insufficient KV space for'):
64
+ assert_enough_kv_space(worker, inputs, num_steps)
65
+
66
+ seq_group_metadata.block_tables = original_block_tables
67
+
68
+
69
+@torch.inference_mode()
70
+def test_same_output_for_single_step():
71
+ """Verify the multi step worker produces the same output as the normal
72
+ worker for num_steps=1.
73
+ """
74
+ seed = 100
75
+ model_name = 'JackFram/llama-68m'
76
+
77
+ block_size = 32
78
+ num_gpu_blocks = 2048 // block_size
79
+ multi_step_worker = create_worker(
80
+ MultiStepWorker,
81
+ model_name,
82
+ block_size,
83
+ num_gpu_blocks,
84
+ seed,
85
+ )
86
+ worker = create_worker(
87
+ Worker,
88
+ model_name,
89
+ block_size,
90
+ num_gpu_blocks,
91
+ seed,
92
+ )
93
+ multi_step_worker.model_runner = worker.model_runner
94
+ multi_step_worker.cache_engine = worker.cache_engine
95
+
96
+ num_steps = 1
97
+
98
+ prompts = [
99
+ [1, 2, 3, 4, 5],
100
+ [6, 7, 8, 9, 10],
101
+ ]
102
+
103
+ final_seq_lens = [len(prompt) + num_steps for prompt in prompts]
104
+
105
+ multi_step_execute_model_data = create_execute_model_data(
106
+ seq_group_metadata_list=create_seq_group_metadata_from_prompts(
107
+ prompts, num_gpu_blocks, block_size,
108
+ final_seq_lens=final_seq_lens))
109
+
110
+ single_step_execute_model_data = create_execute_model_data(
111
+ seq_group_metadata_list=create_seq_group_metadata_from_prompts(
112
+ prompts, num_gpu_blocks, block_size,
113
+ final_seq_lens=final_seq_lens))
114
+
115
+ zero_kv_cache(multi_step_worker.cache_engine)
116
+ set_random_seed(seed)
117
+ actual_output = multi_step_worker.execute_model_multi_step(
118
+ **multi_step_execute_model_data.to_dict(), num_steps=num_steps)
119
+ assert len(actual_output) == num_steps
120
+ actual_output = actual_output[0]
121
+
122
+ zero_kv_cache(worker.cache_engine)
123
+ set_random_seed(seed)
124
+ expected_output = worker.execute_model(
125
+ **single_step_execute_model_data.to_dict(), )
126
+
127
+ actual_token_ids = [
128
+ output.samples[0].output_token for output in actual_output
129
+ ]
130
+ actual_logprobs = [output.samples[0].logprobs for output in actual_output]
131
+
132
+ expected_token_ids = [
133
+ output.samples[0].output_token for output in expected_output
134
+ ]
135
+ expected_logprobs = [
136
+ output.samples[0].logprobs for output in expected_output
137
+ ]
138
+
139
+ assert actual_token_ids == expected_token_ids
140
+
141
+ print(f'{actual_logprobs=}')
142
+ print(f'{expected_logprobs=}')
143
+ assert_logprobs_dict_allclose(actual_logprobs, expected_logprobs)
144
+
145
+
146
+@torch.inference_mode()
147
+def test_same_output_for_multi_step():
148
+ """Verify the multi-step worker produces the same output as the normal
149
+ worker when num_steps > 1. This test runs the multi-step worker once, and
150
+ then runs the worker num_steps times, and compares the output.
151
+ """
152
+ seed = 100
153
+ model_name = 'JackFram/llama-68m'
154
+
155
+ block_size = 16
156
+ num_gpu_blocks = 2048 // block_size
157
+ multi_step_worker = create_worker(
158
+ MultiStepWorker,
159
+ model_name,
160
+ block_size,
161
+ num_gpu_blocks,
162
+ seed,
163
+ )
164
+
165
+ worker = create_worker(
166
+ Worker,
167
+ model_name,
168
+ block_size,
169
+ num_gpu_blocks,
170
+ seed,
171
+ )
172
+
173
+ # Make sure we go over the block boundary.
174
+ num_steps = block_size + 1
175
+
176
+ random.seed(seed)
177
+ prompts = [[
178
+ random.randint(0, 1000) for _ in range(random.randint(10, 20))
179
+ ] for _ in range(10)]
180
+
181
+ final_seq_lens = [len(prompt) + num_steps for prompt in prompts]
182
+
183
+ rand_seeds = list(random.randint(0, 100) for _ in range(num_steps))
184
+ multi_step_worker.execute_model = patch_execute_model_with_seeds(
185
+ multi_step_worker, rand_seeds)
186
+ worker.execute_model = patch_execute_model_with_seeds(worker, rand_seeds)
187
+
188
+ continuations = [[1] for _ in prompts]
189
+ execute_model_data = create_execute_model_data(
190
+ create_seq_group_metadata_from_prompts(
191
+ prompts,
192
+ num_gpu_blocks,
193
+ block_size,
194
+ continuations=continuations,
195
+ final_seq_lens=final_seq_lens), )
196
+
197
+ # Run multi-step.
198
+ zero_kv_cache(multi_step_worker.cache_engine)
199
+ set_random_seed(seed)
200
+ multi_step_output = multi_step_worker.execute_model_multi_step(
201
+ **execute_model_data.to_dict(), num_steps=num_steps)
202
+
203
+ # Run single-step repeatedly.
204
+ zero_kv_cache(worker.cache_engine)
205
+ single_step_output = []
206
+ continuations = [[1] for _ in prompts]
207
+ set_random_seed(seed)
208
+
209
+ for _ in multi_step_output:
210
+
211
+ execute_model_data = create_execute_model_data(
212
+ create_seq_group_metadata_from_prompts(
213
+ prompts,
214
+ num_gpu_blocks,
215
+ block_size,
216
+ continuations=continuations,
217
+ final_seq_lens=final_seq_lens))
218
+
219
+ single_step_output.append(
220
+ worker.execute_model(**execute_model_data.to_dict(), ))
221
+
222
+ # Append output tokens to new sequence data.
223
+ for i, seq_group_output in enumerate(single_step_output[-1]):
224
+ continuations[i].append(seq_group_output.samples[0].output_token)
225
+
226
+ # Get token ids and logprobs for comparison.
227
+ multi_step_output_logprobs = [[] for _ in prompts]
228
+ single_step_output_logprobs = [[] for _ in prompts]
229
+
230
+ multi_step_output_token_ids = [[] for _ in prompts]
231
+ single_step_output_token_ids = [[] for _ in prompts]
232
+ for i, _ in enumerate(prompts):
233
+ for multi_step, single_step in zip(multi_step_output,
234
+ single_step_output):
235
+ multi_step_output_token_ids[i].append(
236
+ multi_step[i].samples[0].output_token)
237
+ single_step_output_token_ids[i].append(
238
+ single_step[i].samples[0].output_token)
239
+
240
+ multi_step_output_logprobs[i].append(
241
+ multi_step[i].samples[0].logprobs)
242
+ single_step_output_logprobs[i].append(
243
+ single_step[i].samples[0].logprobs)
244
+
245
+ # Print per-sequence token ids
246
+ for i, (multi_step_tokens, single_step_tokens) in enumerate(
247
+ zip(multi_step_output_token_ids, single_step_output_token_ids)):
248
+ print(f'{i=} {multi_step_tokens=}')
249
+ print(f'{i=} {single_step_tokens=}')
250
+ print(f'{i=} equal {multi_step_tokens == single_step_tokens}')
251
+
252
+ # Assert token ids are equal.
253
+ for multi_step_tokens, single_step_tokens in zip(
254
+ multi_step_output_token_ids, single_step_output_token_ids):
255
+ assert multi_step_tokens == single_step_tokens
256
+
257
+ # Assert logprobs are equal.
258
+ for multi_step_logprobs, single_step_logprobs in zip(
259
+ multi_step_output_logprobs, single_step_output_logprobs):
260
+ assert_logprobs_dict_allclose(multi_step_logprobs,
261
+ single_step_logprobs)
tests/worker/spec_decode/utils.py
new
+177
@@ -0,0 +1,177 @@
1
+import torch
2
+from typing import List, Optional, Dict
3
+
4
+from vllm.worker.worker import Worker
5
+from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6
+from vllm.engine.arg_utils import EngineArgs
7
+from vllm.sequence import SequenceGroupMetadata, SequenceData
8
+from vllm.sampling_params import SamplingParams
9
+from vllm.worker.cache_engine import CacheEngine
10
+from vllm.model_executor.utils import set_random_seed
11
+from dataclasses import dataclass, fields
12
+
13
+
14
+@dataclass
15
+class ExecuteModelData:
16
+ """Helper data structure which facilitates cleaner tests.
17
+ """
18
+ seq_group_metadata_list: List[SequenceGroupMetadata]
19
+ blocks_to_swap_in: Dict[int, int]
20
+ blocks_to_swap_out: Dict[int, int]
21
+ blocks_to_copy: Dict[int, List[int]]
22
+
23
+ def to_dict(self):
24
+ return dict(
25
+ (field.name, getattr(self, field.name)) for field in fields(self))
26
+
27
+
28
+def round_up_to_next_block(seq_len: int, block_size: int) -> int:
29
+ return (seq_len + block_size - 1) // block_size
30
+
31
+
32
+def create_execute_model_data(
33
+ seq_group_metadata_list: List[SequenceGroupMetadata],
34
+ blocks_to_swap_in: Optional[Dict[int, int]] = None,
35
+ blocks_to_swap_out: Optional[Dict[int, int]] = None,
36
+ blocks_to_copy: Optional[Dict[int, int]] = None,
37
+) -> ExecuteModelData:
38
+ if blocks_to_swap_in is None:
39
+ blocks_to_swap_in = {}
40
+ if blocks_to_swap_out is None:
41
+ blocks_to_swap_out = {}
42
+ if blocks_to_copy is None:
43
+ blocks_to_copy = {}
44
+
45
+ return ExecuteModelData(
46
+ seq_group_metadata_list=seq_group_metadata_list,
47
+ blocks_to_swap_in=blocks_to_swap_in,
48
+ blocks_to_swap_out=blocks_to_swap_out,
49
+ blocks_to_copy=blocks_to_copy,
50
+ )
51
+
52
+
53
+def patch_execute_model_with_seeds(worker: Worker, rand_seeds: List[int]):
54
+ seed_iter = iter(rand_seeds)
55
+ original_execute_model = worker.execute_model
56
+
57
+ def new_execute_model(*args, **kwargs):
58
+ result = original_execute_model(*args, **kwargs)
59
+ set_random_seed(next(seed_iter))
60
+ return result
61
+
62
+ return new_execute_model
63
+
64
+
65
+def zero_kv_cache(cache_engine: CacheEngine):
66
+ assert cache_engine.gpu_cache
67
+ for key_blocks, value_blocks in cache_engine.gpu_cache:
68
+ key_blocks.zero_()
69
+ value_blocks.zero_()
70
+
71
+
72
+def create_worker(cls: type,
73
+ model_name: str,
74
+ block_size: int,
75
+ num_gpu_blocks: int,
76
+ seed: int,
77
+ is_driver_worker: bool = True,
78
+ enforce_eager: bool = True):
79
+ engine_args = EngineArgs(
80
+ model=model_name,
81
+ seed=seed,
82
+ block_size=block_size,
83
+ enforce_eager=enforce_eager,
84
+ )
85
+
86
+ (model_config, cache_config, parallel_config,
87
+ scheduler_config) = engine_args.create_engine_configs()
88
+
89
+ distributed_init_method = get_distributed_init_method(
90
+ get_ip(), get_open_port())
91
+
92
+ worker = cls(
93
+ model_config=model_config,
94
+ parallel_config=parallel_config,
95
+ scheduler_config=scheduler_config,
96
+ local_rank=0,
97
+ rank=0,
98
+ distributed_init_method=distributed_init_method,
99
+ is_driver_worker=is_driver_worker,
100
+ )
101
+
102
+ worker.init_model()
103
+ worker.load_model()
104
+
105
+ cache_config.num_gpu_blocks = num_gpu_blocks
106
+ cache_config.num_cpu_blocks = 0
107
+ worker.init_cache_engine(cache_config)
108
+ worker.warm_up_model()
109
+
110
+ return worker
111
+
112
+
113
+def create_seq_group_metadata_from_prompts(
114
+ prompts: List[List[int]],
115
+ num_gpu_blocks: int,
116
+ block_size: int,
117
+ final_seq_lens: List[int],
118
+ continuations: Optional[List[List[int]]] = None,
119
+ num_tokens_processed: Optional[List[int]] = None,
120
+ seq_ids: Optional[List[int]] = None,
121
+) -> List[SequenceGroupMetadata]:
122
+
123
+ if continuations is None:
124
+ continuations = [[] for _ in prompts]
125
+
126
+ if num_tokens_processed is None:
127
+ # Default to 1 token missing from kv cache for generation sequences.
128
+ num_tokens_processed = []
129
+ for continuation, prompt in zip(continuations, prompts):
130
+ # If prefill, then default to zero tokens processed.
131
+ if not continuation:
132
+ num_tokens_processed.append(0)
133
+ else:
134
+ # If generation, then default to all but one tokens processed.
135
+ num_tokens_processed.append(
136
+ len(continuation) + len(prompt) - 1)
137
+
138
+ if seq_ids is None:
139
+ seq_ids = list(i for i, _ in enumerate(prompts))
140
+
141
+ free_gpu_blocks = list(range(num_gpu_blocks))
142
+
143
+ block_allocations = {
144
+ i: [
145
+ free_gpu_blocks.pop()
146
+ for _ in range(round_up_to_next_block(final_len, block_size))
147
+ ]
148
+ for i, final_len in enumerate(final_seq_lens)
149
+ }
150
+
151
+ return [
152
+ SequenceGroupMetadata(
153
+ request_id=str(i),
154
+ is_prompt=len(cont_token_ids) == 0,
155
+ seq_data={
156
+ i:
157
+ SequenceData(prompt_token_ids=prompt_token_ids[:] +
158
+ cont_token_ids[:])
159
+ },
160
+ sampling_params=SamplingParams(temperature=0.0, ),
161
+ block_tables={i: block_allocations[i][:]},
162
+ ) for i, (prompt_token_ids, cont_token_ids, num_tokens_saved) in
163
+ enumerate(zip(prompts, continuations, num_tokens_processed))
164
+ ]
165
+
166
+
167
+def assert_logprobs_dict_allclose(
168
+ actual_logprobs: List[Dict[int, float]],
169
+ expected_logprobs: List[Dict[int, float]]) -> None:
170
+ for single_step_actual_logprobs, single_step_expected_logprobs in zip(
171
+ actual_logprobs, expected_logprobs):
172
+ assert set(single_step_actual_logprobs.keys()) == set(
173
+ single_step_expected_logprobs.keys())
174
+ for token_id in single_step_actual_logprobs:
175
+ actual = torch.tensor(single_step_actual_logprobs[token_id])
176
+ expected = torch.tensor(single_step_expected_logprobs[token_id])
177
+ assert torch.allclose(actual, expected)
vllm/engine/llm_engine.py
+5
-3
@@ -18,7 +18,7 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
18
SequenceGroupOutput, SequenceOutput, SequenceStatus)
19
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
20
get_tokenizer)
21
-from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port
21
+from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port, get_distributed_init_method
22
23
if ray:
24
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -132,7 +132,8 @@ class LLMEngine:
132
"Ray is required if parallel_config.world_size > 1.")
133
134
self.workers: List[Worker] = []
135
- distributed_init_method = f"tcp://{get_ip()}:{get_open_port()}"
135
+ distributed_init_method = get_distributed_init_method(
136
+ get_ip(), get_open_port())
137
self.driver_worker = Worker(
138
self.model_config,
139
self.parallel_config,
@@ -207,7 +208,8 @@ class LLMEngine:
208
for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
209
worker.set_cuda_visible_devices.remote(node_gpus[node_id])
210
210
- distributed_init_method = f"tcp://{driver_ip}:{get_open_port()}"
211
+ distributed_init_method = get_distributed_init_method(
212
+ driver_ip, get_open_port)
213
214
# Lazy import the Worker to avoid importing torch.cuda/xformers
215
# before CUDA_VISIBLE_DEVICES is set in the Worker
vllm/engine/ray_utils.py
+3
-4
@@ -65,10 +65,9 @@ def initialize_cluster(
65
the default Ray cluster address.
66
67
Returns:
68
- A tuple of (`distributed_init_method`, `placement_group`). The
69
- `distributed_init_method` is the address for initializing the
70
- distributed backend. `placement_group` includes the specification
71
- of the resources for each distributed worker.
68
+ An optional `PlacementGroup`. It includes the specification
69
+ of the resources for each distributed worker. None if Ray is
70
+ not used.
71
"""
72
if parallel_config.worker_use_ray or engine_use_ray:
73
if ray is None:
vllm/model_executor/parallel_utils/parallel_state.py
+25
@@ -83,6 +83,31 @@ def initialize_model_parallel(
83
_PIPELINE_GLOBAL_RANKS = ranks
84
85
86
+def ensure_model_parallel_initialized(
87
+ tensor_model_parallel_size: int,
88
+ pipeline_model_parallel_size: int,
89
+) -> None:
90
+ """Helper to initialize model parallel groups if they are not initialized,
91
+ or ensure tensor-parallel and pipeline-parallel sizes are equal to expected
92
+ values if the model parallel groups are initialized.
93
+ """
94
+ if not model_parallel_is_initialized():
95
+ initialize_model_parallel(tensor_model_parallel_size,
96
+ pipeline_model_parallel_size)
97
+ return
98
+
99
+ assert (
100
+ get_tensor_model_parallel_world_size() == tensor_model_parallel_size
101
+ ), ("tensor parallel group already initialized, but of unexpected size: "
102
+ f"{get_tensor_model_parallel_world_size()=} vs. "
103
+ f"{tensor_model_parallel_size=}")
104
+ assert (get_pipeline_model_parallel_world_size(
105
+ ) == pipeline_model_parallel_size), (
106
+ "pipeline parallel group already initialized, but of unexpected size: "
107
+ f"{get_pipeline_model_parallel_world_size()=} vs. "
108
+ f"{pipeline_model_parallel_size=}")
109
+
110
+
111
def model_parallel_is_initialized():
112
"""Check if tensor and pipeline parallel groups are initialized."""
113
return (_TENSOR_MODEL_PARALLEL_GROUP is not None
vllm/utils.py
+4
@@ -65,6 +65,10 @@ def get_ip() -> str:
65
return s.getsockname()[0]
66
67
68
+def get_distributed_init_method(ip: str, port: int) -> str:
69
+ return f"tcp://{ip}:{port}"
70
+
71
+
72
def get_open_port() -> int:
73
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
74
s.bind(("", 0))
vllm/worker/model_runner.py
+2
-2
@@ -277,8 +277,8 @@ class ModelRunner:
277
input_block_tables[i, :len(block_table)] = block_table
278
block_tables = torch.tensor(input_block_tables, device="cuda")
279
else:
280
- max_block_table_len = (max_context_len + self.block_size -
281
- 1) // self.block_size
280
+ max_block_table_len = max(
281
+ len(block_table) for block_table in block_tables)
282
block_tables = _make_tensor_with_pad(
283
block_tables,
284
max_len=max_block_table_len,
vllm/worker/spec_decode/multi_step_worker.py
new
+178
@@ -0,0 +1,178 @@
1
+from typing import List, Dict
2
+import copy
3
+
4
+import torch
5
+
6
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
+from vllm.worker.worker import Worker
8
+
9
+
10
+class MultiStepWorker(Worker):
11
+ """The MultiStepWorker is equivalent to a Worker except that it allows
12
+ multiple forward passes in a single call, assuming the scheduler has
13
+ allocated enough space to store the additional KV. This reduces overhead
14
+ by invoking the scheduler less.
15
+
16
+ The MultiStepWorker does not support cache swap operations, or beam search.
17
+ Cache swap operations do not require large modifications. On the other hand,
18
+ beam search requires memory allocations during sequence forks and thus
19
+ requires more thought for MultiStepWorker support.
20
+ """
21
+
22
+ @torch.inference_mode()
23
+ def execute_model_multi_step(
24
+ self,
25
+ seq_group_metadata_list: List[SequenceGroupMetadata],
26
+ blocks_to_swap_in: Dict[int, int],
27
+ blocks_to_swap_out: Dict[int, int],
28
+ blocks_to_copy: Dict[int, List[int]],
29
+ num_steps: int,
30
+ ) -> List[SamplerOutput]:
31
+ """Run the model forward pass num_steps times. Returns the list of
32
+ sampler output, one per model forward pass.
33
+ """
34
+ self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
35
+ blocks_to_swap_out, blocks_to_copy)
36
+
37
+ # Shallow copy input data so modifications (such as appending tokens)
38
+ # do not cause side-effects.
39
+ copied_seq_group_metadata_list = self._shallow_copy_inputs(
40
+ seq_group_metadata_list)
41
+
42
+ # Assert enough KV space for num_steps tokens per sequence.
43
+ self._assert_enough_kv_space(seq_group_metadata_list, num_steps)
44
+
45
+ # Run model num_steps times.
46
+ model_outputs = []
47
+ for _ in range(num_steps):
48
+ model_output = super().execute_model(
49
+ seq_group_metadata_list=copied_seq_group_metadata_list,
50
+ blocks_to_swap_in=blocks_to_swap_in,
51
+ blocks_to_swap_out=blocks_to_swap_out,
52
+ blocks_to_copy=blocks_to_copy,
53
+ )
54
+
55
+ self._append_new_tokens(model_output,
56
+ copied_seq_group_metadata_list)
57
+ model_outputs.append(model_output)
58
+
59
+ return model_outputs
60
+
61
+ def _append_new_tokens(
62
+ self, model_output: SamplerOutput,
63
+ seq_group_metadata_list: SequenceGroupMetadata) -> None:
64
+ """Given model output from a single run, append the tokens to the
65
+ sequences. This is normally done outside of the worker, but it is
66
+ required if the worker is to perform multiple forward passes.
67
+ """
68
+ for seq_group_metadata, sequence_group_outputs in zip(
69
+ seq_group_metadata_list, model_output):
70
+ seq_group_metadata.is_prompt = False
71
+
72
+ for seq_output in sequence_group_outputs.samples:
73
+ # NOTE: Beam search is not supported, so we can assume that
74
+ # parent_seq_id == seq_id.
75
+ seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
76
+
77
+ token_id = seq_output.output_token
78
+ token_logprob = seq_output.logprobs[token_id]
79
+
80
+ seq.append_token_id(token_id, token_logprob)
81
+
82
+ def _shallow_copy_inputs(
83
+ self, seq_group_metadata_list: List[SequenceGroupMetadata]
84
+ ) -> List[SequenceGroupMetadata]:
85
+ """Copy input data structures to remove side-effects when input data
86
+ structures are shared with other modules.
87
+
88
+ The multi-step worker must be able to append tokens to sequences after
89
+ a forward pass. This necessitates modification of the data structures
90
+ used by the worker. Since these data structures are shared with other
91
+ parts of vLLM, like the scheduler, we must take care not to introduce
92
+ unexpected side-effects.
93
+
94
+ When Ray is used to orchestrate worker processes (such as when the
95
+ tensor-parallel degree is >1), this is not a problem because the input
96
+ datastructures will be serialized and created anew in the worker
97
+ process.
98
+
99
+ However, when Ray is not used to orchestrate the worker processes (such
100
+ as when the tensor-parallel degree is 1), this is a problem. We avoid
101
+ the problem by shallow-copying the input datastructures (specifically,
102
+ the parts that will change in multiple steps).
103
+ """
104
+
105
+ # Shallow-copy the list of SequenceGroupMetadata. This allows us to
106
+ # append tokens and change is_prompt without external side-effects.
107
+ new_seq_group_metadata_list = []
108
+
109
+ for old_seq_group_metadata in seq_group_metadata_list:
110
+ # We must shallow-copy seq_group_metadata as is_prompt could change.
111
+ seq_group_metadata = copy.copy(old_seq_group_metadata)
112
+ new_seq_group_metadata_list.append(seq_group_metadata)
113
+
114
+ # We must shallow-copy seq_data as we will append token ids
115
+ new_seq_data = {}
116
+ for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
117
+ new_seq_data[seq_id] = copy.copy(old_seq_data)
118
+ new_seq_data[
119
+ seq_id].output_token_ids = old_seq_data.output_token_ids[:]
120
+
121
+ seq_group_metadata.seq_data = new_seq_data
122
+
123
+ return new_seq_group_metadata_list
124
+
125
+ def _assert_enough_kv_space(
126
+ self, seq_group_metadata_list: List[SequenceGroupMetadata],
127
+ num_steps: int) -> None:
128
+ """Assert there are enough physical blocks per sequence to store the
129
+ current KV plus additional KV from num_steps tokens.
130
+ """
131
+ assert self.model_runner.block_size is not None
132
+ for seq_group_metadata in seq_group_metadata_list:
133
+ # Only one seq_id is guaranteed because there is no beam search.
134
+ seq_id = list(seq_group_metadata.seq_data.keys())[0]
135
+ seq = seq_group_metadata.seq_data[seq_id]
136
+
137
+ # After num_steps, the seq len will be the current seq len
138
+ # plus one token per step.
139
+ final_seq_len = seq.get_len() + num_steps
140
+
141
+ # We will have final_seq_len - 1 KV because vLLM saves KV for a
142
+ # token in the iteration after the token was generated.
143
+ required_num_kv_slots = final_seq_len - 1
144
+
145
+ # The allocated number of kv slots is the number of allocated blocks
146
+ # times the number of slots of block.
147
+ number_physical_blocks = len(
148
+ seq_group_metadata.block_tables[seq_id])
149
+ allocated_kv_slots = (number_physical_blocks *
150
+ self.model_runner.block_size)
151
+
152
+ if required_num_kv_slots > allocated_kv_slots:
153
+ request_id = seq_group_metadata.request_id
154
+ raise ValueError(
155
+ "The worker attempted to run "
156
+ f"{num_steps} times but found insufficient KV space for "
157
+ f"{request_id=} {seq_id=}. ({allocated_kv_slots=} "
158
+ f"{required_num_kv_slots=}).")
159
+
160
+ def _raise_if_unsupported(
161
+ self,
162
+ seq_group_metadata_list: List[SequenceGroupMetadata],
163
+ blocks_to_swap_in: Dict[int, int],
164
+ blocks_to_swap_out: Dict[int, int],
165
+ blocks_to_copy: Dict[int, List[int]],
166
+ ) -> None:
167
+ """MultiStepWorker does not yet implement support for cache swap
168
+ operations or beam search.
169
+ """
170
+ if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
171
+ raise NotImplementedError(
172
+ "MultiStepWorker does not support cache operations")
173
+
174
+ if any(
175
+ len(seq_group_metadata.seq_data.keys()) != 1
176
+ for seq_group_metadata in seq_group_metadata_list):
177
+ raise NotImplementedError(
178
+ "MultiStepWorker does not support beam search.")
vllm/worker/worker.py
+3
-3
@@ -11,7 +11,7 @@ from vllm.model_executor import set_random_seed
11
from vllm.model_executor.parallel_utils.communication_op import (
12
broadcast_tensor_dict)
13
from vllm.model_executor.parallel_utils.parallel_state import (
14
- initialize_model_parallel)
14
+ ensure_model_parallel_initialized)
15
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16
from vllm.worker.cache_engine import CacheEngine
17
from vllm.worker.model_runner import ModelRunner
@@ -227,8 +227,8 @@ def _init_distributed_environment(
227
228
# A small all_reduce for warmup.
229
torch.distributed.all_reduce(torch.zeros(1).cuda())
230
- initialize_model_parallel(parallel_config.tensor_parallel_size,
231
- parallel_config.pipeline_parallel_size)
230
+ ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
231
+ parallel_config.pipeline_parallel_size)
232
233
234
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):