@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18bfcdd05c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Speculative decoding 2/9] Multi-step worker for draft model (#2424)

Cade Daniel committed Jan 21, 2024 at 16:31 UTC 18bfcdd05c657e6997b132488e6f4e74307d6cee
11 files changed +658 -12
tests/worker/__init__.py
tests/worker/spec_decode/__init__.py
tests/worker/spec_decode/test_multi_step_worker.py new
+261
@@ -0,0 +1,261 @@
1 +import torch
2 +import random
3 +import pytest
4 +from unittest.mock import MagicMock
5 +
6 +from vllm.worker.spec_decode.multi_step_worker import MultiStepWorker
7 +from vllm.worker.worker import Worker
8 +from vllm.model_executor.utils import set_random_seed
9 +
10 +from .utils import (create_execute_model_data, create_worker,
11 + create_seq_group_metadata_from_prompts, zero_kv_cache,
12 + patch_execute_model_with_seeds,
13 + assert_logprobs_dict_allclose)
14 +
15 +
16 +@pytest.mark.parametrize('num_steps', list(range(1, 17)))
17 +def test_assert_enough_kv_space(num_steps: int):
18 + """Test that the multi step worker checks for sufficient space in the KV
19 + cache. It should throw if it cannot run all the steps.
20 + """
21 + block_size = 16
22 + num_gpu_blocks = 2048 // block_size
23 +
24 + prompts = [
25 + list(range(block_size * 3)),
26 + list(range(block_size * 2)),
27 + ]
28 +
29 + prev_output_tokens = [
30 + list(range(block_size * 1)),
31 + list(range(block_size * 2)),
32 + ]
33 +
34 + final_seq_lens = [
35 + len(prompt + output) + num_steps
36 + for prompt, output in zip(prompts, prev_output_tokens)
37 + ]
38 +
39 + inputs = create_seq_group_metadata_from_prompts(
40 + prompts,
41 + num_gpu_blocks,
42 + block_size,
43 + final_seq_lens,
44 + continuations=prev_output_tokens)
45 +
46 + assert_enough_kv_space = MultiStepWorker._assert_enough_kv_space # pylint: disable=protected-access
47 + worker = MagicMock()
48 + worker.model_runner.block_size = block_size
49 +
50 + for seq_group_metadata in inputs:
51 + original_block_tables = seq_group_metadata.block_tables
52 +
53 + # No exception.
54 + assert_enough_kv_space(worker, inputs, num_steps)
55 +
56 + seq_group_metadata.block_tables = {
57 + seq_id: []
58 + for seq_id, physical_blocks in original_block_tables.items()
59 + }
60 +
61 + # Expect exception.
62 + with pytest.raises(ValueError,
63 + match='times but found insufficient KV space for'):
64 + assert_enough_kv_space(worker, inputs, num_steps)
65 +
66 + seq_group_metadata.block_tables = original_block_tables
67 +
68 +
69 +@torch.inference_mode()
70 +def test_same_output_for_single_step():
71 + """Verify the multi step worker produces the same output as the normal
72 + worker for num_steps=1.
73 + """
74 + seed = 100
75 + model_name = 'JackFram/llama-68m'
76 +
77 + block_size = 32
78 + num_gpu_blocks = 2048 // block_size
79 + multi_step_worker = create_worker(
80 + MultiStepWorker,
81 + model_name,
82 + block_size,
83 + num_gpu_blocks,
84 + seed,
85 + )
86 + worker = create_worker(
87 + Worker,
88 + model_name,
89 + block_size,
90 + num_gpu_blocks,
91 + seed,
92 + )
93 + multi_step_worker.model_runner = worker.model_runner
94 + multi_step_worker.cache_engine = worker.cache_engine
95 +
96 + num_steps = 1
97 +
98 + prompts = [
99 + [1, 2, 3, 4, 5],
100 + [6, 7, 8, 9, 10],
101 + ]
102 +
103 + final_seq_lens = [len(prompt) + num_steps for prompt in prompts]
104 +
105 + multi_step_execute_model_data = create_execute_model_data(
106 + seq_group_metadata_list=create_seq_group_metadata_from_prompts(
107 + prompts, num_gpu_blocks, block_size,
108 + final_seq_lens=final_seq_lens))
109 +
110 + single_step_execute_model_data = create_execute_model_data(
111 + seq_group_metadata_list=create_seq_group_metadata_from_prompts(
112 + prompts, num_gpu_blocks, block_size,
113 + final_seq_lens=final_seq_lens))
114 +
115 + zero_kv_cache(multi_step_worker.cache_engine)
116 + set_random_seed(seed)
117 + actual_output = multi_step_worker.execute_model_multi_step(
118 + **multi_step_execute_model_data.to_dict(), num_steps=num_steps)
119 + assert len(actual_output) == num_steps
120 + actual_output = actual_output[0]
121 +
122 + zero_kv_cache(worker.cache_engine)
123 + set_random_seed(seed)
124 + expected_output = worker.execute_model(
125 + **single_step_execute_model_data.to_dict(), )
126 +
127 + actual_token_ids = [
128 + output.samples[0].output_token for output in actual_output
129 + ]
130 + actual_logprobs = [output.samples[0].logprobs for output in actual_output]
131 +
132 + expected_token_ids = [
133 + output.samples[0].output_token for output in expected_output
134 + ]
135 + expected_logprobs = [
136 + output.samples[0].logprobs for output in expected_output
137 + ]
138 +
139 + assert actual_token_ids == expected_token_ids
140 +
141 + print(f'{actual_logprobs=}')
142 + print(f'{expected_logprobs=}')
143 + assert_logprobs_dict_allclose(actual_logprobs, expected_logprobs)
144 +
145 +
146 +@torch.inference_mode()
147 +def test_same_output_for_multi_step():
148 + """Verify the multi-step worker produces the same output as the normal
149 + worker when num_steps > 1. This test runs the multi-step worker once, and
150 + then runs the worker num_steps times, and compares the output.
151 + """
152 + seed = 100
153 + model_name = 'JackFram/llama-68m'
154 +
155 + block_size = 16
156 + num_gpu_blocks = 2048 // block_size
157 + multi_step_worker = create_worker(
158 + MultiStepWorker,
159 + model_name,
160 + block_size,
161 + num_gpu_blocks,
162 + seed,
163 + )
164 +
165 + worker = create_worker(
166 + Worker,
167 + model_name,
168 + block_size,
169 + num_gpu_blocks,
170 + seed,
171 + )
172 +
173 + # Make sure we go over the block boundary.
174 + num_steps = block_size + 1
175 +
176 + random.seed(seed)
177 + prompts = [[
178 + random.randint(0, 1000) for _ in range(random.randint(10, 20))
179 + ] for _ in range(10)]
180 +
181 + final_seq_lens = [len(prompt) + num_steps for prompt in prompts]
182 +
183 + rand_seeds = list(random.randint(0, 100) for _ in range(num_steps))
184 + multi_step_worker.execute_model = patch_execute_model_with_seeds(
185 + multi_step_worker, rand_seeds)
186 + worker.execute_model = patch_execute_model_with_seeds(worker, rand_seeds)
187 +
188 + continuations = [[1] for _ in prompts]
189 + execute_model_data = create_execute_model_data(
190 + create_seq_group_metadata_from_prompts(
191 + prompts,
192 + num_gpu_blocks,
193 + block_size,
194 + continuations=continuations,
195 + final_seq_lens=final_seq_lens), )
196 +
197 + # Run multi-step.
198 + zero_kv_cache(multi_step_worker.cache_engine)
199 + set_random_seed(seed)
200 + multi_step_output = multi_step_worker.execute_model_multi_step(
201 + **execute_model_data.to_dict(), num_steps=num_steps)
202 +
203 + # Run single-step repeatedly.
204 + zero_kv_cache(worker.cache_engine)
205 + single_step_output = []
206 + continuations = [[1] for _ in prompts]
207 + set_random_seed(seed)
208 +
209 + for _ in multi_step_output:
210 +
211 + execute_model_data = create_execute_model_data(
212 + create_seq_group_metadata_from_prompts(
213 + prompts,
214 + num_gpu_blocks,
215 + block_size,
216 + continuations=continuations,
217 + final_seq_lens=final_seq_lens))
218 +
219 + single_step_output.append(
220 + worker.execute_model(**execute_model_data.to_dict(), ))
221 +
222 + # Append output tokens to new sequence data.
223 + for i, seq_group_output in enumerate(single_step_output[-1]):
224 + continuations[i].append(seq_group_output.samples[0].output_token)
225 +
226 + # Get token ids and logprobs for comparison.
227 + multi_step_output_logprobs = [[] for _ in prompts]
228 + single_step_output_logprobs = [[] for _ in prompts]
229 +
230 + multi_step_output_token_ids = [[] for _ in prompts]
231 + single_step_output_token_ids = [[] for _ in prompts]
232 + for i, _ in enumerate(prompts):
233 + for multi_step, single_step in zip(multi_step_output,
234 + single_step_output):
235 + multi_step_output_token_ids[i].append(
236 + multi_step[i].samples[0].output_token)
237 + single_step_output_token_ids[i].append(
238 + single_step[i].samples[0].output_token)
239 +
240 + multi_step_output_logprobs[i].append(
241 + multi_step[i].samples[0].logprobs)
242 + single_step_output_logprobs[i].append(
243 + single_step[i].samples[0].logprobs)
244 +
245 + # Print per-sequence token ids
246 + for i, (multi_step_tokens, single_step_tokens) in enumerate(
247 + zip(multi_step_output_token_ids, single_step_output_token_ids)):
248 + print(f'{i=} {multi_step_tokens=}')
249 + print(f'{i=} {single_step_tokens=}')
250 + print(f'{i=} equal {multi_step_tokens == single_step_tokens}')
251 +
252 + # Assert token ids are equal.
253 + for multi_step_tokens, single_step_tokens in zip(
254 + multi_step_output_token_ids, single_step_output_token_ids):
255 + assert multi_step_tokens == single_step_tokens
256 +
257 + # Assert logprobs are equal.
258 + for multi_step_logprobs, single_step_logprobs in zip(
259 + multi_step_output_logprobs, single_step_output_logprobs):
260 + assert_logprobs_dict_allclose(multi_step_logprobs,
261 + single_step_logprobs)
tests/worker/spec_decode/utils.py new
+177
@@ -0,0 +1,177 @@
1 +import torch
2 +from typing import List, Optional, Dict
3 +
4 +from vllm.worker.worker import Worker
5 +from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6 +from vllm.engine.arg_utils import EngineArgs
7 +from vllm.sequence import SequenceGroupMetadata, SequenceData
8 +from vllm.sampling_params import SamplingParams
9 +from vllm.worker.cache_engine import CacheEngine
10 +from vllm.model_executor.utils import set_random_seed
11 +from dataclasses import dataclass, fields
12 +
13 +
14 +@dataclass
15 +class ExecuteModelData:
16 + """Helper data structure which facilitates cleaner tests.
17 + """
18 + seq_group_metadata_list: List[SequenceGroupMetadata]
19 + blocks_to_swap_in: Dict[int, int]
20 + blocks_to_swap_out: Dict[int, int]
21 + blocks_to_copy: Dict[int, List[int]]
22 +
23 + def to_dict(self):
24 + return dict(
25 + (field.name, getattr(self, field.name)) for field in fields(self))
26 +
27 +
28 +def round_up_to_next_block(seq_len: int, block_size: int) -> int:
29 + return (seq_len + block_size - 1) // block_size
30 +
31 +
32 +def create_execute_model_data(
33 + seq_group_metadata_list: List[SequenceGroupMetadata],
34 + blocks_to_swap_in: Optional[Dict[int, int]] = None,
35 + blocks_to_swap_out: Optional[Dict[int, int]] = None,
36 + blocks_to_copy: Optional[Dict[int, int]] = None,
37 +) -> ExecuteModelData:
38 + if blocks_to_swap_in is None:
39 + blocks_to_swap_in = {}
40 + if blocks_to_swap_out is None:
41 + blocks_to_swap_out = {}
42 + if blocks_to_copy is None:
43 + blocks_to_copy = {}
44 +
45 + return ExecuteModelData(
46 + seq_group_metadata_list=seq_group_metadata_list,
47 + blocks_to_swap_in=blocks_to_swap_in,
48 + blocks_to_swap_out=blocks_to_swap_out,
49 + blocks_to_copy=blocks_to_copy,
50 + )
51 +
52 +
53 +def patch_execute_model_with_seeds(worker: Worker, rand_seeds: List[int]):
54 + seed_iter = iter(rand_seeds)
55 + original_execute_model = worker.execute_model
56 +
57 + def new_execute_model(*args, **kwargs):
58 + result = original_execute_model(*args, **kwargs)
59 + set_random_seed(next(seed_iter))
60 + return result
61 +
62 + return new_execute_model
63 +
64 +
65 +def zero_kv_cache(cache_engine: CacheEngine):
66 + assert cache_engine.gpu_cache
67 + for key_blocks, value_blocks in cache_engine.gpu_cache:
68 + key_blocks.zero_()
69 + value_blocks.zero_()
70 +
71 +
72 +def create_worker(cls: type,
73 + model_name: str,
74 + block_size: int,
75 + num_gpu_blocks: int,
76 + seed: int,
77 + is_driver_worker: bool = True,
78 + enforce_eager: bool = True):
79 + engine_args = EngineArgs(
80 + model=model_name,
81 + seed=seed,
82 + block_size=block_size,
83 + enforce_eager=enforce_eager,
84 + )
85 +
86 + (model_config, cache_config, parallel_config,
87 + scheduler_config) = engine_args.create_engine_configs()
88 +
89 + distributed_init_method = get_distributed_init_method(
90 + get_ip(), get_open_port())
91 +
92 + worker = cls(
93 + model_config=model_config,
94 + parallel_config=parallel_config,
95 + scheduler_config=scheduler_config,
96 + local_rank=0,
97 + rank=0,
98 + distributed_init_method=distributed_init_method,
99 + is_driver_worker=is_driver_worker,
100 + )
101 +
102 + worker.init_model()
103 + worker.load_model()
104 +
105 + cache_config.num_gpu_blocks = num_gpu_blocks
106 + cache_config.num_cpu_blocks = 0
107 + worker.init_cache_engine(cache_config)
108 + worker.warm_up_model()
109 +
110 + return worker
111 +
112 +
113 +def create_seq_group_metadata_from_prompts(
114 + prompts: List[List[int]],
115 + num_gpu_blocks: int,
116 + block_size: int,
117 + final_seq_lens: List[int],
118 + continuations: Optional[List[List[int]]] = None,
119 + num_tokens_processed: Optional[List[int]] = None,
120 + seq_ids: Optional[List[int]] = None,
121 +) -> List[SequenceGroupMetadata]:
122 +
123 + if continuations is None:
124 + continuations = [[] for _ in prompts]
125 +
126 + if num_tokens_processed is None:
127 + # Default to 1 token missing from kv cache for generation sequences.
128 + num_tokens_processed = []
129 + for continuation, prompt in zip(continuations, prompts):
130 + # If prefill, then default to zero tokens processed.
131 + if not continuation:
132 + num_tokens_processed.append(0)
133 + else:
134 + # If generation, then default to all but one tokens processed.
135 + num_tokens_processed.append(
136 + len(continuation) + len(prompt) - 1)
137 +
138 + if seq_ids is None:
139 + seq_ids = list(i for i, _ in enumerate(prompts))
140 +
141 + free_gpu_blocks = list(range(num_gpu_blocks))
142 +
143 + block_allocations = {
144 + i: [
145 + free_gpu_blocks.pop()
146 + for _ in range(round_up_to_next_block(final_len, block_size))
147 + ]
148 + for i, final_len in enumerate(final_seq_lens)
149 + }
150 +
151 + return [
152 + SequenceGroupMetadata(
153 + request_id=str(i),
154 + is_prompt=len(cont_token_ids) == 0,
155 + seq_data={
156 + i:
157 + SequenceData(prompt_token_ids=prompt_token_ids[:] +
158 + cont_token_ids[:])
159 + },
160 + sampling_params=SamplingParams(temperature=0.0, ),
161 + block_tables={i: block_allocations[i][:]},
162 + ) for i, (prompt_token_ids, cont_token_ids, num_tokens_saved) in
163 + enumerate(zip(prompts, continuations, num_tokens_processed))
164 + ]
165 +
166 +
167 +def assert_logprobs_dict_allclose(
168 + actual_logprobs: List[Dict[int, float]],
169 + expected_logprobs: List[Dict[int, float]]) -> None:
170 + for single_step_actual_logprobs, single_step_expected_logprobs in zip(
171 + actual_logprobs, expected_logprobs):
172 + assert set(single_step_actual_logprobs.keys()) == set(
173 + single_step_expected_logprobs.keys())
174 + for token_id in single_step_actual_logprobs:
175 + actual = torch.tensor(single_step_actual_logprobs[token_id])
176 + expected = torch.tensor(single_step_expected_logprobs[token_id])
177 + assert torch.allclose(actual, expected)
vllm/engine/llm_engine.py
+5 -3
@@ -18,7 +18,7 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
18 SequenceGroupOutput, SequenceOutput, SequenceStatus)
19 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
20 get_tokenizer)
21 -from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port
21 +from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port, get_distributed_init_method
22
23 if ray:
24 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -132,7 +132,8 @@ class LLMEngine:
132 "Ray is required if parallel_config.world_size > 1.")
133
134 self.workers: List[Worker] = []
135 - distributed_init_method = f"tcp://{get_ip()}:{get_open_port()}"
135 + distributed_init_method = get_distributed_init_method(
136 + get_ip(), get_open_port())
137 self.driver_worker = Worker(
138 self.model_config,
139 self.parallel_config,
@@ -207,7 +208,8 @@ class LLMEngine:
208 for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
209 worker.set_cuda_visible_devices.remote(node_gpus[node_id])
210
210 - distributed_init_method = f"tcp://{driver_ip}:{get_open_port()}"
211 + distributed_init_method = get_distributed_init_method(
212 + driver_ip, get_open_port)
213
214 # Lazy import the Worker to avoid importing torch.cuda/xformers
215 # before CUDA_VISIBLE_DEVICES is set in the Worker
vllm/engine/ray_utils.py
+3 -4
@@ -65,10 +65,9 @@ def initialize_cluster(
65 the default Ray cluster address.
66
67 Returns:
68 - A tuple of (`distributed_init_method`, `placement_group`). The
69 - `distributed_init_method` is the address for initializing the
70 - distributed backend. `placement_group` includes the specification
71 - of the resources for each distributed worker.
68 + An optional `PlacementGroup`. It includes the specification
69 + of the resources for each distributed worker. None if Ray is
70 + not used.
71 """
72 if parallel_config.worker_use_ray or engine_use_ray:
73 if ray is None:
vllm/model_executor/parallel_utils/parallel_state.py
+25
@@ -83,6 +83,31 @@ def initialize_model_parallel(
83 _PIPELINE_GLOBAL_RANKS = ranks
84
85
86 +def ensure_model_parallel_initialized(
87 + tensor_model_parallel_size: int,
88 + pipeline_model_parallel_size: int,
89 +) -> None:
90 + """Helper to initialize model parallel groups if they are not initialized,
91 + or ensure tensor-parallel and pipeline-parallel sizes are equal to expected
92 + values if the model parallel groups are initialized.
93 + """
94 + if not model_parallel_is_initialized():
95 + initialize_model_parallel(tensor_model_parallel_size,
96 + pipeline_model_parallel_size)
97 + return
98 +
99 + assert (
100 + get_tensor_model_parallel_world_size() == tensor_model_parallel_size
101 + ), ("tensor parallel group already initialized, but of unexpected size: "
102 + f"{get_tensor_model_parallel_world_size()=} vs. "
103 + f"{tensor_model_parallel_size=}")
104 + assert (get_pipeline_model_parallel_world_size(
105 + ) == pipeline_model_parallel_size), (
106 + "pipeline parallel group already initialized, but of unexpected size: "
107 + f"{get_pipeline_model_parallel_world_size()=} vs. "
108 + f"{pipeline_model_parallel_size=}")
109 +
110 +
111 def model_parallel_is_initialized():
112 """Check if tensor and pipeline parallel groups are initialized."""
113 return (_TENSOR_MODEL_PARALLEL_GROUP is not None
vllm/utils.py
+4
@@ -65,6 +65,10 @@ def get_ip() -> str:
65 return s.getsockname()[0]
66
67
68 +def get_distributed_init_method(ip: str, port: int) -> str:
69 + return f"tcp://{ip}:{port}"
70 +
71 +
72 def get_open_port() -> int:
73 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
74 s.bind(("", 0))
vllm/worker/model_runner.py
+2 -2
@@ -277,8 +277,8 @@ class ModelRunner:
277 input_block_tables[i, :len(block_table)] = block_table
278 block_tables = torch.tensor(input_block_tables, device="cuda")
279 else:
280 - max_block_table_len = (max_context_len + self.block_size -
281 - 1) // self.block_size
280 + max_block_table_len = max(
281 + len(block_table) for block_table in block_tables)
282 block_tables = _make_tensor_with_pad(
283 block_tables,
284 max_len=max_block_table_len,
vllm/worker/spec_decode/multi_step_worker.py new
+178
@@ -0,0 +1,178 @@
1 +from typing import List, Dict
2 +import copy
3 +
4 +import torch
5 +
6 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 +from vllm.worker.worker import Worker
8 +
9 +
10 +class MultiStepWorker(Worker):
11 + """The MultiStepWorker is equivalent to a Worker except that it allows
12 + multiple forward passes in a single call, assuming the scheduler has
13 + allocated enough space to store the additional KV. This reduces overhead
14 + by invoking the scheduler less.
15 +
16 + The MultiStepWorker does not support cache swap operations, or beam search.
17 + Cache swap operations do not require large modifications. On the other hand,
18 + beam search requires memory allocations during sequence forks and thus
19 + requires more thought for MultiStepWorker support.
20 + """
21 +
22 + @torch.inference_mode()
23 + def execute_model_multi_step(
24 + self,
25 + seq_group_metadata_list: List[SequenceGroupMetadata],
26 + blocks_to_swap_in: Dict[int, int],
27 + blocks_to_swap_out: Dict[int, int],
28 + blocks_to_copy: Dict[int, List[int]],
29 + num_steps: int,
30 + ) -> List[SamplerOutput]:
31 + """Run the model forward pass num_steps times. Returns the list of
32 + sampler output, one per model forward pass.
33 + """
34 + self._raise_if_unsupported(seq_group_metadata_list, blocks_to_swap_in,
35 + blocks_to_swap_out, blocks_to_copy)
36 +
37 + # Shallow copy input data so modifications (such as appending tokens)
38 + # do not cause side-effects.
39 + copied_seq_group_metadata_list = self._shallow_copy_inputs(
40 + seq_group_metadata_list)
41 +
42 + # Assert enough KV space for num_steps tokens per sequence.
43 + self._assert_enough_kv_space(seq_group_metadata_list, num_steps)
44 +
45 + # Run model num_steps times.
46 + model_outputs = []
47 + for _ in range(num_steps):
48 + model_output = super().execute_model(
49 + seq_group_metadata_list=copied_seq_group_metadata_list,
50 + blocks_to_swap_in=blocks_to_swap_in,
51 + blocks_to_swap_out=blocks_to_swap_out,
52 + blocks_to_copy=blocks_to_copy,
53 + )
54 +
55 + self._append_new_tokens(model_output,
56 + copied_seq_group_metadata_list)
57 + model_outputs.append(model_output)
58 +
59 + return model_outputs
60 +
61 + def _append_new_tokens(
62 + self, model_output: SamplerOutput,
63 + seq_group_metadata_list: SequenceGroupMetadata) -> None:
64 + """Given model output from a single run, append the tokens to the
65 + sequences. This is normally done outside of the worker, but it is
66 + required if the worker is to perform multiple forward passes.
67 + """
68 + for seq_group_metadata, sequence_group_outputs in zip(
69 + seq_group_metadata_list, model_output):
70 + seq_group_metadata.is_prompt = False
71 +
72 + for seq_output in sequence_group_outputs.samples:
73 + # NOTE: Beam search is not supported, so we can assume that
74 + # parent_seq_id == seq_id.
75 + seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
76 +
77 + token_id = seq_output.output_token
78 + token_logprob = seq_output.logprobs[token_id]
79 +
80 + seq.append_token_id(token_id, token_logprob)
81 +
82 + def _shallow_copy_inputs(
83 + self, seq_group_metadata_list: List[SequenceGroupMetadata]
84 + ) -> List[SequenceGroupMetadata]:
85 + """Copy input data structures to remove side-effects when input data
86 + structures are shared with other modules.
87 +
88 + The multi-step worker must be able to append tokens to sequences after
89 + a forward pass. This necessitates modification of the data structures
90 + used by the worker. Since these data structures are shared with other
91 + parts of vLLM, like the scheduler, we must take care not to introduce
92 + unexpected side-effects.
93 +
94 + When Ray is used to orchestrate worker processes (such as when the
95 + tensor-parallel degree is >1), this is not a problem because the input
96 + datastructures will be serialized and created anew in the worker
97 + process.
98 +
99 + However, when Ray is not used to orchestrate the worker processes (such
100 + as when the tensor-parallel degree is 1), this is a problem. We avoid
101 + the problem by shallow-copying the input datastructures (specifically,
102 + the parts that will change in multiple steps).
103 + """
104 +
105 + # Shallow-copy the list of SequenceGroupMetadata. This allows us to
106 + # append tokens and change is_prompt without external side-effects.
107 + new_seq_group_metadata_list = []
108 +
109 + for old_seq_group_metadata in seq_group_metadata_list:
110 + # We must shallow-copy seq_group_metadata as is_prompt could change.
111 + seq_group_metadata = copy.copy(old_seq_group_metadata)
112 + new_seq_group_metadata_list.append(seq_group_metadata)
113 +
114 + # We must shallow-copy seq_data as we will append token ids
115 + new_seq_data = {}
116 + for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
117 + new_seq_data[seq_id] = copy.copy(old_seq_data)
118 + new_seq_data[
119 + seq_id].output_token_ids = old_seq_data.output_token_ids[:]
120 +
121 + seq_group_metadata.seq_data = new_seq_data
122 +
123 + return new_seq_group_metadata_list
124 +
125 + def _assert_enough_kv_space(
126 + self, seq_group_metadata_list: List[SequenceGroupMetadata],
127 + num_steps: int) -> None:
128 + """Assert there are enough physical blocks per sequence to store the
129 + current KV plus additional KV from num_steps tokens.
130 + """
131 + assert self.model_runner.block_size is not None
132 + for seq_group_metadata in seq_group_metadata_list:
133 + # Only one seq_id is guaranteed because there is no beam search.
134 + seq_id = list(seq_group_metadata.seq_data.keys())[0]
135 + seq = seq_group_metadata.seq_data[seq_id]
136 +
137 + # After num_steps, the seq len will be the current seq len
138 + # plus one token per step.
139 + final_seq_len = seq.get_len() + num_steps
140 +
141 + # We will have final_seq_len - 1 KV because vLLM saves KV for a
142 + # token in the iteration after the token was generated.
143 + required_num_kv_slots = final_seq_len - 1
144 +
145 + # The allocated number of kv slots is the number of allocated blocks
146 + # times the number of slots of block.
147 + number_physical_blocks = len(
148 + seq_group_metadata.block_tables[seq_id])
149 + allocated_kv_slots = (number_physical_blocks *
150 + self.model_runner.block_size)
151 +
152 + if required_num_kv_slots > allocated_kv_slots:
153 + request_id = seq_group_metadata.request_id
154 + raise ValueError(
155 + "The worker attempted to run "
156 + f"{num_steps} times but found insufficient KV space for "
157 + f"{request_id=} {seq_id=}. ({allocated_kv_slots=} "
158 + f"{required_num_kv_slots=}).")
159 +
160 + def _raise_if_unsupported(
161 + self,
162 + seq_group_metadata_list: List[SequenceGroupMetadata],
163 + blocks_to_swap_in: Dict[int, int],
164 + blocks_to_swap_out: Dict[int, int],
165 + blocks_to_copy: Dict[int, List[int]],
166 + ) -> None:
167 + """MultiStepWorker does not yet implement support for cache swap
168 + operations or beam search.
169 + """
170 + if any([blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy]):
171 + raise NotImplementedError(
172 + "MultiStepWorker does not support cache operations")
173 +
174 + if any(
175 + len(seq_group_metadata.seq_data.keys()) != 1
176 + for seq_group_metadata in seq_group_metadata_list):
177 + raise NotImplementedError(
178 + "MultiStepWorker does not support beam search.")
vllm/worker/worker.py
+3 -3
@@ -11,7 +11,7 @@ from vllm.model_executor import set_random_seed
11 from vllm.model_executor.parallel_utils.communication_op import (
12 broadcast_tensor_dict)
13 from vllm.model_executor.parallel_utils.parallel_state import (
14 - initialize_model_parallel)
14 + ensure_model_parallel_initialized)
15 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16 from vllm.worker.cache_engine import CacheEngine
17 from vllm.worker.model_runner import ModelRunner
@@ -227,8 +227,8 @@ def _init_distributed_environment(
227
228 # A small all_reduce for warmup.
229 torch.distributed.all_reduce(torch.zeros(1).cuda())
230 - initialize_model_parallel(parallel_config.tensor_parallel_size,
231 - parallel_config.pipeline_parallel_size)
230 + ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
231 + parallel_config.pipeline_parallel_size)
232
233
234 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):