@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fd4ea8ef5c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Use NCCL instead of ray for control-plane communication to remove serialization overhead (#2221)

Zhuohan Li committed Jan 4, 2024 at 03:30 UTC fd4ea8ef5c17a8b991107402a414f6ed355d854d
34 files changed +519 -257
docs/source/models/adding_model.rst
+3 -4
@@ -58,11 +58,10 @@ Next, you need to rewrite the :code:`forward` methods of your model by following
58 + positions: torch.Tensor,
59 + kv_caches: List[KVCache],
60 + input_metadata: InputMetadata,
61 - + cache_events: Optional[List[torch.cuda.Event]],
62 - +) -> SamplerOutput:
61 + +) -> Optional[SamplerOutput]:
62
64 -3. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
65 -4. Replace the attention operation with either :code:`PagedAttention`, :code:`PagedAttentionWithRoPE`, or :code:`PagedAttentionWithALiBi` depending on the model's architecture.
63 +1. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
64 +2. Replace the attention operation with either :code:`PagedAttention`, :code:`PagedAttentionWithRoPE`, or :code:`PagedAttentionWithALiBi` depending on the model's architecture.
65
66 .. note::
67 Currently, vLLM supports the basic multi-head attention mechanism and its variant with rotary positional embeddings.
requirements-rocm.txt
-2
@@ -3,8 +3,6 @@ typing-extensions>=4.8.0
3 starlette
4 psutil
5 ray >= 2.5.1
6 -pandas # Required for Ray data.
7 -pyarrow # Required for Ray data.
6 sentencepiece # Required for LLaMA tokenizer.
7 numpy
8 tokenizers>=0.15.0
requirements.txt
-2
@@ -1,8 +1,6 @@
1 ninja # For faster builds.
2 psutil
3 ray >= 2.5.1
4 -pandas # Required for Ray data.
5 -pyarrow # Required for Ray data.
4 sentencepiece # Required for LLaMA tokenizer.
5 numpy
6 torch == 2.1.2
tests/async_engine/test_api_server.py
+9 -4
@@ -8,11 +8,11 @@ import pytest
8 import requests
9
10
11 -def _query_server(prompt: str) -> dict:
11 +def _query_server(prompt: str, max_tokens: int = 5) -> dict:
12 response = requests.post("http://localhost:8000/generate",
13 json={
14 "prompt": prompt,
15 - "max_tokens": 100,
15 + "max_tokens": max_tokens,
16 "temperature": 0,
17 "ignore_eos": True
18 })
@@ -20,6 +20,10 @@ def _query_server(prompt: str) -> dict:
20 return response.json()
21
22
23 +def _query_server_long(prompt: str) -> dict:
24 + return _query_server(prompt, max_tokens=500)
25 +
26 +
27 @pytest.fixture
28 def api_server():
29 script_path = Path(__file__).parent.joinpath(
@@ -68,10 +72,11 @@ def test_api_server(api_server):
72 for result in pool.map(_query_server, prompts):
73 assert result
74
75 + with Pool(32) as pool:
76 # Cancel requests
77 prompts = ["canceled requests"] * 100
73 - pool.map_async(_query_server, prompts)
74 - time.sleep(0.001)
78 + pool.map_async(_query_server_long, prompts)
79 + time.sleep(0.01)
80 pool.terminate()
81 pool.join()
82
tests/kernels/test_cache.py
+12 -12
@@ -49,12 +49,13 @@ def test_copy_blocks(
49 src_blocks = random.sample(range(num_blocks), num_mappings)
50 remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
51 dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)
52 - block_mapping = {}
52 + copy_src = []
53 + copy_dst = []
54 for i in range(num_mappings):
54 - src = src_blocks[i]
55 - dst1 = dst_blocks[2 * i]
56 - dst2 = dst_blocks[2 * i + 1]
57 - block_mapping[src] = [dst1, dst2]
55 + copy_src.append(src_blocks[i])
56 + copy_dst.append(dst_blocks[2 * i])
57 + copy_src.append(src_blocks[i])
58 + copy_dst.append(dst_blocks[2 * i + 1])
59
60 # Create the KV caches.
61 key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
@@ -66,15 +67,14 @@ def test_copy_blocks(
67 cloned_value_caches = [value_cache.clone() for value_cache in value_caches]
68
69 # Call the copy blocks kernel.
69 - cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
70 + cache_ops.copy_blocks(key_caches, value_caches, copy_src, copy_dst)
71
72 # Run the reference implementation.
72 - for src, dsts in block_mapping.items():
73 - for dst in dsts:
74 - for cloned_key_cache in cloned_key_caches:
75 - cloned_key_cache[dst].copy_(cloned_key_cache[src])
76 - for cloned_value_cache in cloned_value_caches:
77 - cloned_value_cache[dst].copy_(cloned_value_cache[src])
73 + for src, dst in zip(copy_src, copy_dst):
74 + for cloned_key_cache in cloned_key_caches:
75 + cloned_key_cache[dst].copy_(cloned_key_cache[src])
76 + for cloned_value_cache in cloned_value_caches:
77 + cloned_value_cache[dst].copy_(cloned_value_cache[src])
78
79 # Compare the results.
80 for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
tests/worker/test_model_runner.py
+3 -2
@@ -33,8 +33,9 @@ def test_prepare_prompt():
33 expected_selected_token_indices.append(selected_token_start_idx +
34 prompt_len - 1)
35 selected_token_start_idx += max_seq_len
36 - input_tokens, input_positions, _ = model_runner._prepare_prompt(
37 - seq_group_metadata_list)
36 + input_tokens, input_positions, _, return_prompt_lens = (
37 + model_runner._prepare_prompt(seq_group_metadata_list))
38 + assert return_prompt_lens == prompt_lens
39 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
40 prompt_lens)
41 assert input_tokens.shape == (batch_size, max_seq_len)
vllm/engine/async_llm_engine.py
+33 -28
@@ -185,14 +185,21 @@ class _AsyncLLMEngine(LLMEngine):
185 """
186 seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
187
188 - # Execute the model.
189 - output = (await self._run_workers_async(
190 - "execute_model",
191 - seq_group_metadata_list=seq_group_metadata_list,
192 - blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
193 - blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
194 - blocks_to_copy=scheduler_outputs.blocks_to_copy,
195 - )) if not scheduler_outputs.is_empty() else []
188 + if not scheduler_outputs.is_empty():
189 + # Execute the model.
190 + all_outputs = await self._run_workers_async(
191 + "execute_model",
192 + driver_kwargs={
193 + "seq_group_metadata_list": seq_group_metadata_list,
194 + "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
195 + "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
196 + "blocks_to_copy": scheduler_outputs.blocks_to_copy,
197 + })
198 +
199 + # Only the driver worker returns the sampling results.
200 + output = all_outputs[0]
201 + else:
202 + output = []
203
204 return self._process_model_outputs(output, scheduler_outputs)
205
@@ -200,30 +207,29 @@ class _AsyncLLMEngine(LLMEngine):
207 self,
208 method: str,
209 *args,
203 - get_all_outputs: bool = False,
210 + driver_args: Optional[List[Any]] = None,
211 + driver_kwargs: Optional[Dict[str, Any]] = None,
212 **kwargs,
213 ) -> Any:
214 """Runs the given method on all workers."""
215 coros = []
208 - for worker in self.workers:
209 - if self.parallel_config.worker_use_ray:
210 - coros.append(
211 - worker.execute_method.remote(method, *args, **kwargs))
212 - else:
213 - executor = getattr(worker, method)
214 - coros.append(asyncio.get_event_loop().run_in_executor(
215 - None, partial(executor, *args, **kwargs)))
216
217 - all_outputs = await asyncio.gather(*coros)
217 + if driver_args is None:
218 + driver_args = args
219 + if driver_kwargs is None:
220 + driver_kwargs = kwargs
221
219 - if get_all_outputs:
220 - return all_outputs
222 + # Run the driver worker asynchronously.
223 + driver_executor = getattr(self.driver_worker, method)
224 + coros.append(asyncio.get_event_loop().run_in_executor(
225 + None, partial(driver_executor, *driver_args, **driver_kwargs)))
226
222 - # Make sure all workers have the same results.
223 - output = all_outputs[0]
224 - for other_output in all_outputs[1:]:
225 - assert output == other_output
226 - return output
227 + # Run the ray workers asynchronously.
228 + for worker in self.workers:
229 + coros.append(worker.execute_method.remote(method, *args, **kwargs))
230 +
231 + all_outputs = await asyncio.gather(*coros)
232 + return all_outputs
233
234
235 class AsyncLLMEngine:
@@ -488,13 +494,12 @@ class AsyncLLMEngine:
494 engine_configs = engine_args.create_engine_configs()
495 parallel_config = engine_configs[2]
496 # Initialize the cluster.
491 - distributed_init_method, placement_group = initialize_cluster(
492 - parallel_config, engine_args.engine_use_ray)
497 + placement_group = initialize_cluster(parallel_config,
498 + engine_args.engine_use_ray)
499 # Create the async LLM engine.
500 engine = cls(parallel_config.worker_use_ray,
501 engine_args.engine_use_ray,
502 *engine_configs,
497 - distributed_init_method,
503 placement_group,
504 log_requests=not engine_args.disable_log_requests,
505 log_stats=not engine_args.disable_log_stats,
vllm/engine/llm_engine.py
+138 -100
@@ -1,8 +1,9 @@
1 import copy
2 +from collections import defaultdict
3 import os
4 import time
4 -from functools import partial
5 -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union
5 +from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
6 + Union)
7
8 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig)
@@ -17,10 +18,9 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
18 SequenceGroupOutput, SequenceOutput, SequenceStatus)
19 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
20 get_tokenizer)
20 -from vllm.utils import Counter
21 +from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port
22
23 if ray:
23 - from ray.air.util.torch_dist import init_torch_dist_process_group
24 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
25
26 if TYPE_CHECKING:
@@ -53,8 +53,6 @@ class LLMEngine:
53 management.
54 parallel_config: The configuration related to distributed execution.
55 scheduler_config: The configuration related to the request scheduler.
56 - distributed_init_method: The initialization method for distributed
57 - execution. See `torch.distributed.init_process_group` for details.
56 placement_group: Ray placement group for distributed execution.
57 Required for distributed execution.
58 log_stats: Whether to log statistics.
@@ -66,7 +64,6 @@ class LLMEngine:
64 cache_config: CacheConfig,
65 parallel_config: ParallelConfig,
66 scheduler_config: SchedulerConfig,
69 - distributed_init_method: str,
67 placement_group: Optional["PlacementGroup"],
68 log_stats: bool,
69 ) -> None:
@@ -111,7 +108,7 @@ class LLMEngine:
108 os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
109 self._init_workers_ray(placement_group)
110 else:
114 - self._init_workers(distributed_init_method)
111 + self._init_workers()
112
113 # Profile the memory usage and initialize the cache.
114 self._init_cache()
@@ -126,7 +123,7 @@ class LLMEngine:
123 # List of (timestamp, num_tokens)
124 self.num_generation_tokens: List[Tuple[float, int]] = []
125
129 - def _init_workers(self, distributed_init_method: str):
126 + def _init_workers(self):
127 # Lazy import the Worker to avoid importing torch.cuda/xformers
128 # before CUDA_VISIBLE_DEVICES is set in the Worker
129 from vllm.worker.worker import Worker
@@ -135,70 +132,122 @@ class LLMEngine:
132 "Ray is required if parallel_config.world_size > 1.")
133
134 self.workers: List[Worker] = []
138 - worker = Worker(
135 + distributed_init_method = f"tcp://{get_ip()}:{get_open_port()}"
136 + self.driver_worker = Worker(
137 self.model_config,
138 self.parallel_config,
139 self.scheduler_config,
142 - 0,
143 - distributed_init_method,
144 - )
145 - self.workers.append(worker)
146 - self._run_workers(
147 - "init_model",
148 - get_all_outputs=True,
149 - )
150 - self._run_workers(
151 - "load_model",
152 - get_all_outputs=True,
153 - max_concurrent_workers=self.parallel_config.
154 - max_parallel_loading_workers,
140 + local_rank=0,
141 + rank=0,
142 + distributed_init_method=distributed_init_method,
143 + is_driver_worker=True,
144 )
145 + self._run_workers("init_model")
146 + self._run_workers("load_model")
147
148 def _init_workers_ray(self, placement_group: "PlacementGroup",
149 **ray_remote_kwargs):
159 - # Lazy import the Worker to avoid importing torch.cuda/xformers
160 - # before CUDA_VISIBLE_DEVICES is set in the Worker
161 - from vllm.worker.worker import Worker
150 + if self.parallel_config.tensor_parallel_size == 1:
151 + num_gpus = self.cache_config.gpu_memory_utilization
152 + else:
153 + num_gpus = 1
154
163 - self.workers: List[Worker] = []
164 - for bundle in placement_group.bundle_specs:
155 + self.driver_dummy_worker: RayWorkerVllm = None
156 + self.workers: List[RayWorkerVllm] = []
157 +
158 + driver_ip = get_ip()
159 + for bundle_id, bundle in enumerate(placement_group.bundle_specs):
160 if not bundle.get("GPU", 0):
161 continue
167 - if self.parallel_config.tensor_parallel_size == 1:
168 - num_gpus = self.cache_config.gpu_memory_utilization
169 - else:
170 - num_gpus = 1
162 + scheduling_strategy = PlacementGroupSchedulingStrategy(
163 + placement_group=placement_group,
164 + placement_group_capture_child_tasks=True,
165 + placement_group_bundle_index=bundle_id,
166 + )
167 worker = ray.remote(
168 num_cpus=0,
169 num_gpus=num_gpus,
174 - scheduling_strategy=PlacementGroupSchedulingStrategy(
175 - placement_group=placement_group,
176 - placement_group_capture_child_tasks=True),
170 + scheduling_strategy=scheduling_strategy,
171 **ray_remote_kwargs,
172 )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
179 - self.workers.append(worker)
173 +
174 + worker_ip = ray.get(worker.get_node_ip.remote())
175 + if worker_ip == driver_ip and self.driver_dummy_worker is None:
176 + # If the worker is on the same node as the driver, we use it
177 + # as the resource holder for the driver process.
178 + self.driver_dummy_worker = worker
179 + else:
180 + self.workers.append(worker)
181 +
182 + if self.driver_dummy_worker is None:
183 + raise ValueError(
184 + "Ray does not allocate any GPUs on the driver node. Consider "
185 + "adjusting the Ray placement group or running the driver on a "
186 + "GPU node.")
187 +
188 + driver_node_id, driver_gpu_ids = ray.get(
189 + self.driver_dummy_worker.get_node_and_gpu_ids.remote())
190 + worker_node_and_gpu_ids = ray.get(
191 + [worker.get_node_and_gpu_ids.remote() for worker in self.workers])
192 +
193 + node_workers = defaultdict(list)
194 + node_gpus = defaultdict(list)
195 +
196 + node_workers[driver_node_id].append(0)
197 + node_gpus[driver_node_id].extend(driver_gpu_ids)
198 + for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids,
199 + start=1):
200 + node_workers[node_id].append(i)
201 + node_gpus[node_id].extend(gpu_ids)
202 + for node_id, gpu_ids in node_gpus.items():
203 + node_gpus[node_id] = sorted(gpu_ids)
204 +
205 + # Set CUDA_VISIBLE_DEVICES for the driver.
206 + set_cuda_visible_devices(node_gpus[driver_node_id])
207 + for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
208 + worker.set_cuda_visible_devices.remote(node_gpus[node_id])
209 +
210 + distributed_init_method = f"tcp://{driver_ip}:{get_open_port()}"
211 +
212 + # Lazy import the Worker to avoid importing torch.cuda/xformers
213 + # before CUDA_VISIBLE_DEVICES is set in the Worker
214 + from vllm.worker.worker import Worker
215
216 # Initialize torch distributed process group for the workers.
182 - init_torch_dist_process_group(self.workers, backend="nccl")
217 model_config = copy.deepcopy(self.model_config)
218 parallel_config = copy.deepcopy(self.parallel_config)
219 scheduler_config = copy.deepcopy(self.scheduler_config)
186 - self._run_workers("init_worker",
187 - get_all_outputs=True,
188 - worker_init_fn=lambda: Worker(
189 - model_config,
190 - parallel_config,
191 - scheduler_config,
192 - None,
193 - None,
194 - ))
195 - self._run_workers(
196 - "init_model",
197 - get_all_outputs=True,
220 +
221 + for rank, (worker, (node_id,
222 + _)) in enumerate(zip(self.workers,
223 + worker_node_and_gpu_ids),
224 + start=1):
225 + local_rank = node_workers[node_id].index(rank)
226 + worker.init_worker.remote(
227 + lambda rank=rank, local_rank=local_rank: Worker(
228 + model_config,
229 + parallel_config,
230 + scheduler_config,
231 + local_rank,
232 + rank,
233 + distributed_init_method,
234 + ))
235 +
236 + driver_rank = 0
237 + driver_local_rank = node_workers[driver_node_id].index(driver_rank)
238 + self.driver_worker = Worker(
239 + model_config,
240 + parallel_config,
241 + scheduler_config,
242 + driver_local_rank,
243 + driver_rank,
244 + distributed_init_method,
245 + is_driver_worker=True,
246 )
247 +
248 + self._run_workers("init_model")
249 self._run_workers(
250 "load_model",
201 - get_all_outputs=True,
251 max_concurrent_workers=self.parallel_config.
252 max_parallel_loading_workers,
253 )
@@ -212,7 +261,6 @@ class LLMEngine:
261 # Get the maximum number of blocks that can be allocated on GPU and CPU.
262 num_blocks = self._run_workers(
263 "profile_num_available_blocks",
215 - get_all_outputs=True,
264 block_size=self.cache_config.block_size,
265 gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
266 cpu_swap_space=self.cache_config.swap_space_bytes,
@@ -256,11 +304,9 @@ class LLMEngine:
304 engine_configs = engine_args.create_engine_configs()
305 parallel_config = engine_configs[2]
306 # Initialize the cluster.
259 - distributed_init_method, placement_group = initialize_cluster(
260 - parallel_config)
307 + placement_group = initialize_cluster(parallel_config)
308 # Create the LLM engine.
309 engine = cls(*engine_configs,
263 - distributed_init_method,
310 placement_group,
311 log_stats=not engine_args.disable_log_stats)
312 return engine
@@ -577,14 +623,21 @@ class LLMEngine:
623 """
624 seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
625
580 - # Execute the model.
581 - output = self._run_workers(
582 - "execute_model",
583 - seq_group_metadata_list=seq_group_metadata_list,
584 - blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
585 - blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
586 - blocks_to_copy=scheduler_outputs.blocks_to_copy,
587 - ) if not scheduler_outputs.is_empty() else []
626 + if not scheduler_outputs.is_empty():
627 + # Execute the model.
628 + all_outputs = self._run_workers(
629 + "execute_model",
630 + driver_kwargs={
631 + "seq_group_metadata_list": seq_group_metadata_list,
632 + "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
633 + "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
634 + "blocks_to_copy": scheduler_outputs.blocks_to_copy,
635 + })
636 +
637 + # Only the driver worker returns the sampling results.
638 + output = all_outputs[0]
639 + else:
640 + output = []
641
642 return self._process_model_outputs(output, scheduler_outputs)
643
@@ -712,53 +765,38 @@ class LLMEngine:
765 seq.status = SequenceStatus.FINISHED_STOPPED
766 return
767
715 - def _run_workers_in_batch(
716 - self,
717 - workers,
718 - method: str,
719 - *args,
720 - **kwargs,
721 - ):
722 - all_outputs = []
723 - for worker in workers:
724 - if self.parallel_config.worker_use_ray:
725 - executor = partial(worker.execute_method.remote, method)
726 - else:
727 - executor = getattr(worker, method)
728 -
729 - output = executor(*args, **kwargs)
730 - all_outputs.append(output)
731 - if self.parallel_config.worker_use_ray:
732 - all_outputs = ray.get(all_outputs)
733 - return all_outputs
734 -
768 def _run_workers(
769 self,
770 method: str,
771 *args,
739 - get_all_outputs: bool = False,
772 + driver_args: Optional[List[Any]] = None,
773 + driver_kwargs: Optional[Dict[str, Any]] = None,
774 max_concurrent_workers: Optional[int] = None,
775 **kwargs,
776 ) -> Any:
777 """Runs the given method on all workers."""
744 - all_outputs = []
778 +
779 if max_concurrent_workers:
746 - work_groups = [
747 - self.workers[i:i + max_concurrent_workers]
748 - for i in range(0, len(self.workers), max_concurrent_workers)
749 - ]
750 - else:
751 - work_groups = [self.workers]
780 + raise NotImplementedError(
781 + "max_concurrent_workers is not supported yet.")
782 +
783 + # Start the ray workers first.
784 + ray_worker_outputs = [
785 + worker.execute_method.remote(method, *args, **kwargs)
786 + for worker in self.workers
787 + ]
788 +
789 + if driver_args is None:
790 + driver_args = args
791 + if driver_kwargs is None:
792 + driver_kwargs = kwargs
793
753 - for workers in work_groups:
754 - all_outputs.extend(
755 - self._run_workers_in_batch(workers, method, *args, **kwargs))
794 + # Start the driver worker after all the ray workers.
795 + driver_worker_output = getattr(self.driver_worker,
796 + method)(*driver_args, **driver_kwargs)
797
757 - if get_all_outputs:
758 - return all_outputs
798 + # Get the results of the ray workers.
799 + if self.workers:
800 + ray_worker_outputs = ray.get(ray_worker_outputs)
801
760 - # Make sure all workers have the same results.
761 - output = all_outputs[0]
762 - for other_output in all_outputs[1:]:
763 - assert output == other_output
764 - return output
802 + return [driver_worker_output] + ray_worker_outputs
vllm/engine/ray_utils.py
+22 -15
@@ -1,16 +1,15 @@
1 -from typing import Optional, Tuple, TYPE_CHECKING
1 +from typing import Optional, List, Tuple, TYPE_CHECKING
2
3 from vllm.config import ParallelConfig
4 from vllm.logger import init_logger
5 -from vllm.utils import get_open_port, is_hip
5 +from vllm.utils import is_hip, set_cuda_visible_devices, get_ip
6
7 logger = init_logger(__name__)
8
9 try:
10 import ray
11 - from ray.air.util.torch_dist import TorchDistributedWorker
11
13 - class RayWorkerVllm(TorchDistributedWorker):
12 + class RayWorkerVllm:
13 """Ray wrapper for vllm.worker.Worker, allowing Worker to be
14 lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
15
@@ -30,12 +29,22 @@ try:
29 executor = getattr(self, method)
30 return executor(*args, **kwargs)
31
32 + def get_node_ip(self) -> str:
33 + return get_ip()
34 +
35 + def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
36 + node_id = ray.get_runtime_context().get_node_id()
37 + gpu_ids = ray.get_gpu_ids()
38 + return node_id, gpu_ids
39 +
40 + def set_cuda_visible_devices(self, device_ids) -> None:
41 + set_cuda_visible_devices(device_ids)
42 +
43 except ImportError as e:
44 logger.warning(f"Failed to import Ray with {e!r}. "
45 "For distributed inference, please install Ray with "
46 "`pip install ray pandas pyarrow`.")
47 ray = None
38 - TorchDistributedWorker = None
48 RayWorkerVllm = None
49
50 if TYPE_CHECKING:
@@ -75,13 +84,11 @@ def initialize_cluster(
84 ray.init(address=ray_address, ignore_reinit_error=True)
85
86 if not parallel_config.worker_use_ray:
78 - # Initialize cluster locally.
79 - port = get_open_port()
80 - # We need to setup the distributed init method to make sure
81 - # the distributed megatron code (e.g., get world size) works correctly.
82 - distributed_init_method = f"tcp://localhost:{port}"
83 - return distributed_init_method, None
87 + assert parallel_config.world_size == 1, (
88 + "Ray is required if parallel_config.world_size > 1.")
89 + return None
90
91 + # Create placement group for worker processes
92 current_placement_group = ray.util.get_current_placement_group()
93 if current_placement_group:
94 # We are in a placement group
@@ -106,12 +113,12 @@ def initialize_cluster(
113 "The number of required GPUs exceeds the total number of "
114 "available GPUs in the cluster.")
115 # Create a new placement group
109 - current_placement_group = ray.util.placement_group([{
110 - "GPU": 1
111 - }] * parallel_config.world_size)
116 + placement_group_specs = ([{"GPU": 1}] * parallel_config.world_size)
117 + current_placement_group = ray.util.placement_group(
118 + placement_group_specs)
119 # Wait until PG is ready - this will block until all
120 # requested resources are available, and will timeout
121 # if they cannot be provisioned.
122 ray.get(current_placement_group.ready(), timeout=1800)
123
117 - return None, current_placement_group
124 + return current_placement_group
vllm/model_executor/input_metadata.py
+4 -5
@@ -1,4 +1,4 @@
1 -from typing import List, Optional
1 +from typing import Optional
2
3 import torch
4
@@ -16,28 +16,27 @@ class InputMetadata:
16
17 def __init__(
18 self,
19 - prompt_lens: List[int],
19 + is_prompt: bool,
20 slot_mapping: torch.Tensor,
21 max_context_len: Optional[int],
22 context_lens: Optional[torch.Tensor],
23 block_tables: Optional[torch.Tensor],
24 use_cuda_graph: bool,
25 ) -> None:
26 - self.prompt_lens = prompt_lens
26 + self.is_prompt = is_prompt
27 self.max_context_len = max_context_len
28 self.slot_mapping = slot_mapping
29 self.context_lens = context_lens
30 self.block_tables = block_tables
31 self.use_cuda_graph = use_cuda_graph
32
33 - self.is_prompt = len(prompt_lens) > 0
33 # Set during the execution of the first attention op.
34 # FIXME(woosuk): This is a hack.
35 self.attn_bias = None
36
37 def __repr__(self) -> str:
38 return ("InputMetadata("
40 - f"prompt_lens={self.prompt_lens}, "
39 + f"is_prompt={self.is_prompt}, "
40 f"max_context_len={self.max_context_len}, "
41 f"slot_mapping={self.slot_mapping}, "
42 f"context_lens={self.context_lens}, "
vllm/model_executor/layers/sampler.py
+14 -5
@@ -5,7 +5,7 @@ import torch
5 import torch.nn as nn
6
7 from vllm.model_executor.parallel_utils.communication_op import (
8 - tensor_model_parallel_all_gather)
8 + tensor_model_parallel_gather)
9 from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
@@ -37,7 +37,7 @@ class Sampler(nn.Module):
37 hidden_states: torch.Tensor,
38 sampling_metadata: SamplingMetadata,
39 embedding_bias: Optional[torch.Tensor] = None,
40 - ) -> SamplerOutput:
40 + ) -> Optional[SamplerOutput]:
41 # Get the hidden states that we use for sampling.
42 hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)
43
@@ -45,6 +45,14 @@ class Sampler(nn.Module):
45 logits = _get_logits(hidden_states, embedding, embedding_bias,
46 self.vocab_size)
47
48 + # Only perform sampling in the driver worker.
49 + # Note: `_get_logits` is still distributed across TP workers because
50 + # the `embedding` weight is distributed across TP workers.
51 + # TODO(zhuohan): Change the get_logits part to a separate stage.
52 + if not sampling_metadata.perform_sampling:
53 + return None
54 +
55 + assert logits is not None
56 _, vocab_size = logits.shape
57
58 # Apply logits processors (if any).
@@ -92,14 +100,15 @@ class Sampler(nn.Module):
100
101 def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
102 embedding_bias: Optional[torch.Tensor],
95 - vocab_size: int) -> torch.Tensor:
103 + vocab_size: int) -> Optional[torch.Tensor]:
104 # Get the logits for the next tokens.
105 logits = torch.matmul(hidden_states, embedding.t())
106 if embedding_bias is not None:
107 logits += embedding_bias
100 - logits = tensor_model_parallel_all_gather(logits)
108 + logits = tensor_model_parallel_gather(logits)
109 # Remove paddings in vocab (if any).
102 - logits = logits[:, :vocab_size]
110 + if logits is not None:
111 + logits = logits[:, :vocab_size]
112 return logits
113
114
vllm/model_executor/models/aquila.py
+1 -1
@@ -298,7 +298,7 @@ class AquilaForCausalLM(nn.Module):
298 self,
299 hidden_states: torch.Tensor,
300 sampling_metadata: SamplingMetadata,
301 - ) -> SamplerOutput:
301 + ) -> Optional[SamplerOutput]:
302 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303 sampling_metadata)
304 return next_tokens
vllm/model_executor/models/baichuan.py
+1 -1
@@ -313,7 +313,7 @@ class BaiChuanBaseForCausalLM(nn.Module):
313 self,
314 hidden_states: torch.Tensor,
315 sampling_metadata: SamplingMetadata,
316 - ) -> SamplerOutput:
316 + ) -> Optional[SamplerOutput]:
317 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
318 sampling_metadata)
319 return next_tokens
vllm/model_executor/models/bloom.py
+1 -1
@@ -290,7 +290,7 @@ class BloomForCausalLM(nn.Module):
290 self,
291 hidden_states: torch.Tensor,
292 sampling_metadata: SamplingMetadata,
293 - ) -> SamplerOutput:
293 + ) -> Optional[SamplerOutput]:
294 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
295 sampling_metadata)
296 return next_tokens
vllm/model_executor/models/chatglm.py
+1 -1
@@ -349,7 +349,7 @@ class ChatGLMForCausalLM(nn.Module):
349 self,
350 hidden_states: torch.Tensor,
351 sampling_metadata: SamplingMetadata,
352 - ) -> SamplerOutput:
352 + ) -> Optional[SamplerOutput]:
353 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
354 sampling_metadata)
355 return next_tokens
vllm/model_executor/models/falcon.py
+1 -1
@@ -394,7 +394,7 @@ class FalconForCausalLM(nn.Module):
394 self,
395 hidden_states: torch.Tensor,
396 sampling_metadata: SamplingMetadata,
397 - ) -> SamplerOutput:
397 + ) -> Optional[SamplerOutput]:
398 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
399 sampling_metadata)
400 return next_tokens
vllm/model_executor/models/gpt2.py
+1 -1
@@ -235,7 +235,7 @@ class GPT2LMHeadModel(nn.Module):
235 self,
236 hidden_states: torch.Tensor,
237 sampling_metadata: SamplingMetadata,
238 - ) -> SamplerOutput:
238 + ) -> Optional[SamplerOutput]:
239 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
240 sampling_metadata)
241 return next_tokens
vllm/model_executor/models/gpt_bigcode.py
+1 -1
@@ -254,7 +254,7 @@ class GPTBigCodeForCausalLM(nn.Module):
254 self,
255 hidden_states: torch.Tensor,
256 sampling_metadata: SamplingMetadata,
257 - ) -> SamplerOutput:
257 + ) -> Optional[SamplerOutput]:
258 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
259 sampling_metadata)
260 return next_tokens
vllm/model_executor/models/gpt_j.py
+1 -1
@@ -240,7 +240,7 @@ class GPTJForCausalLM(nn.Module):
240 self,
241 hidden_states: torch.Tensor,
242 sampling_metadata: SamplingMetadata,
243 - ) -> SamplerOutput:
243 + ) -> Optional[SamplerOutput]:
244 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
245 sampling_metadata, self.lm_head.bias)
246 return next_tokens
vllm/model_executor/models/gpt_neox.py
+1 -1
@@ -255,7 +255,7 @@ class GPTNeoXForCausalLM(nn.Module):
255 self,
256 hidden_states: torch.Tensor,
257 sampling_metadata: SamplingMetadata,
258 - ) -> SamplerOutput:
258 + ) -> Optional[SamplerOutput]:
259 next_tokens = self.sampler(self.embed_out.weight, hidden_states,
260 sampling_metadata)
261 return next_tokens
vllm/model_executor/models/internlm.py
+1 -1
@@ -255,7 +255,7 @@ class InternLMForCausalLM(nn.Module):
255 self,
256 hidden_states: torch.Tensor,
257 sampling_metadata: SamplingMetadata,
258 - ) -> SamplerOutput:
258 + ) -> Optional[SamplerOutput]:
259 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260 sampling_metadata)
261 return next_tokens
vllm/model_executor/models/llama.py
+1 -1
@@ -291,7 +291,7 @@ class LlamaForCausalLM(nn.Module):
291 self,
292 hidden_states: torch.Tensor,
293 sampling_metadata: SamplingMetadata,
294 - ) -> SamplerOutput:
294 + ) -> Optional[SamplerOutput]:
295 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
296 sampling_metadata)
297 return next_tokens
vllm/model_executor/models/mistral.py
+1 -1
@@ -287,7 +287,7 @@ class MistralForCausalLM(nn.Module):
287 self,
288 hidden_states: torch.Tensor,
289 sampling_metadata: SamplingMetadata,
290 - ) -> SamplerOutput:
290 + ) -> Optional[SamplerOutput]:
291 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
292 sampling_metadata)
293 return next_tokens
vllm/model_executor/models/mixtral.py
+2 -2
@@ -320,7 +320,7 @@ class MixtralModel(nn.Module):
320 positions: torch.Tensor,
321 kv_caches: List[KVCache],
322 input_metadata: InputMetadata,
323 - ) -> SamplerOutput:
323 + ) -> torch.Tensor:
324 hidden_states = self.embed_tokens(input_ids)
325 residual = None
326 for i in range(len(self.layers)):
@@ -361,7 +361,7 @@ class MixtralForCausalLM(nn.Module):
361 self,
362 hidden_states: Optional[torch.Tensor],
363 sampling_metadata: SamplingMetadata,
364 - ) -> SamplerOutput:
364 + ) -> Optional[SamplerOutput]:
365 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
366 sampling_metadata)
367 return next_tokens
vllm/model_executor/models/mpt.py
+1 -1
@@ -276,7 +276,7 @@ class MPTForCausalLM(nn.Module):
276 self,
277 hidden_states: torch.Tensor,
278 sampling_metadata: SamplingMetadata,
279 - ) -> SamplerOutput:
279 + ) -> Optional[SamplerOutput]:
280 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
281 sampling_metadata)
282 return next_tokens
vllm/model_executor/models/opt.py
+1 -1
@@ -309,7 +309,7 @@ class OPTForCausalLM(nn.Module):
309 self,
310 hidden_states: torch.Tensor,
311 sampling_metadata: SamplingMetadata,
312 - ) -> SamplerOutput:
312 + ) -> Optional[SamplerOutput]:
313 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
314 sampling_metadata)
315 return next_tokens
vllm/model_executor/models/phi_1_5.py
+1 -1
@@ -280,7 +280,7 @@ class PhiForCausalLM(nn.Module):
280 self,
281 hidden_states: torch.Tensor,
282 sampling_metadata: SamplingMetadata,
283 - ) -> SamplerOutput:
283 + ) -> Optional[SamplerOutput]:
284 head = self.lm_head.linear
285 next_tokens = self.sampler(head.weight, hidden_states,
286 sampling_metadata, head.bias)
vllm/model_executor/models/qwen.py
+1 -1
@@ -247,7 +247,7 @@ class QWenLMHeadModel(nn.Module):
247 self,
248 hidden_states: torch.Tensor,
249 sampling_metadata: SamplingMetadata,
250 - ) -> SamplerOutput:
250 + ) -> Optional[SamplerOutput]:
251 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
252 sampling_metadata)
253 return next_tokens
vllm/model_executor/models/yi.py
+1 -1
@@ -286,7 +286,7 @@ class YiForCausalLM(nn.Module):
286 self,
287 hidden_states: torch.Tensor,
288 sampling_metadata: SamplingMetadata,
289 - ) -> SamplerOutput:
289 + ) -> Optional[SamplerOutput]:
290 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291 sampling_metadata)
292 return next_tokens
vllm/model_executor/parallel_utils/communication_op.py
+59
@@ -1,6 +1,7 @@
1 import torch
2
3 from vllm.model_executor.parallel_utils.parallel_state import (
4 + get_tensor_model_parallel_rank,
5 get_tensor_model_parallel_world_size,
6 get_tensor_model_parallel_group,
7 )
@@ -45,3 +46,61 @@ def tensor_model_parallel_all_gather(input_, dim=-1):
46 (world_size * input_size[dim], ) +
47 input_size[dim + 1:])
48 return output_tensor
49 +
50 +
51 +def tensor_model_parallel_gather(input_, dst=0, dim=-1):
52 + """Gather the input tensor across model parallel group.
53 +
54 + NOTE: We assume that the input tensor is on the same device across
55 + all the ranks.
56 + """
57 + world_size = get_tensor_model_parallel_world_size()
58 + # Bypass the function if we are using only 1 GPU.
59 + if world_size == 1:
60 + return input_
61 + assert -input_.dim() <= dim < input_.dim(), (
62 + f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
63 + if dim < 0:
64 + # Convert negative dim to positive.
65 + dim += input_.dim()
66 + # Allocate output tensor.
67 + if get_tensor_model_parallel_rank() == dst:
68 + gather_list = [torch.empty_like(input_) for _ in range(world_size)]
69 + else:
70 + gather_list = None
71 + # Gather.
72 + torch.distributed.gather(input_,
73 + gather_list,
74 + dst=dst,
75 + group=get_tensor_model_parallel_group())
76 + if get_tensor_model_parallel_rank() == dst:
77 + output_tensor = torch.cat(gather_list, dim=dim)
78 + else:
79 + output_tensor = None
80 + return output_tensor
81 +
82 +
83 +def broadcast(input_, src=0):
84 + """Broadcast the input tensor."""
85 + world_size = torch.distributed.get_world_size()
86 + assert 0 <= src < world_size, f"Invalid src rank ({src})"
87 +
88 + # Bypass the function if we are using only 1 GPU.
89 + if world_size == 1:
90 + return input_
91 + # Broadcast.
92 + torch.distributed.broadcast(input_, src=src)
93 + return input_
94 +
95 +
96 +def broadcast_object_list(obj_list, src=0):
97 + """Broadcast the input object list."""
98 + world_size = torch.distributed.get_world_size()
99 + assert 0 <= src < world_size, f"Invalid src rank ({src})"
100 +
101 + # Bypass the function if we are using only 1 GPU.
102 + if world_size == 1:
103 + return obj_list
104 + # Broadcast.
105 + torch.distributed.broadcast_object_list(obj_list, src=src)
106 + return obj_list
vllm/model_executor/sampling_metadata.py
+14 -8
@@ -1,5 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import Dict, List, Tuple
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5
@@ -18,24 +18,29 @@ class SamplingMetadata:
18 seq_data: Seq_id -> SequenceData.
19 prompt_lens: Lengths of prompts.
20 selected_token_indices: Token indices selected for sampling.
21 - categorized_sample_indices: SamplingType -> token indicies to sample.
21 + categorized_sample_indices: SamplingType -> token indices to sample.
22 + perform_sampling: Whether to perform sampling. This option is used to
23 + make the sampling only happens in the driver worker, and disable
24 + sampling in other worker processes.
25 """
26
27 def __init__(
28 self,
26 - seq_groups: List[Tuple[List[int], SamplingParams]],
27 - seq_data: Dict[int, SequenceData],
28 - prompt_lens: List[int],
29 + seq_groups: Optional[List[Tuple[List[int], SamplingParams]]],
30 + seq_data: Optional[Dict[int, SequenceData]],
31 + prompt_lens: Optional[List[int]],
32 selected_token_indices: torch.Tensor,
30 - categorized_sample_indices: Dict[SamplingType, torch.Tensor],
33 + categorized_sample_indices: Optional[Dict[SamplingType, torch.Tensor]],
34 + perform_sampling: bool = True,
35 ) -> None:
36 self.seq_groups = seq_groups
37 self.seq_data = seq_data
38 self.prompt_lens = prompt_lens
39 self.selected_token_indices = selected_token_indices
40 self.categorized_sample_indices = categorized_sample_indices
41 + self.perform_sampling = perform_sampling
42
38 - self.num_prompts = len(prompt_lens)
43 + self.num_prompts = len(prompt_lens) if prompt_lens is not None else 0
44
45 def __repr__(self) -> str:
46 return (
@@ -44,7 +49,8 @@ class SamplingMetadata:
49 f"seq_data={self.seq_data}, "
50 f"prompt_lens={self.prompt_lens}, "
51 f"selected_token_indices={self.selected_token_indices}, "
47 - f"categorized_sample_indices={self.categorized_sample_indices})")
52 + f"categorized_sample_indices={self.categorized_sample_indices}), "
53 + f"perform_sampling={self.perform_sampling})")
54
55
56 @dataclass
vllm/utils.py
+11 -1
@@ -1,7 +1,9 @@
1 import enum
2 +import os
3 import socket
4 import uuid
5 from platform import uname
6 +from typing import List
7
8 import psutil
9 import torch
@@ -55,7 +57,15 @@ def in_wsl() -> bool:
57 return "microsoft" in " ".join(uname()).lower()
58
59
58 -def get_open_port():
60 +def get_ip() -> str:
61 + return socket.gethostbyname(socket.gethostname())
62 +
63 +
64 +def get_open_port() -> int:
65 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
66 s.bind(("", 0))
67 return s.getsockname()[1]
68 +
69 +
70 +def set_cuda_visible_devices(device_ids: List[int]) -> None:
71 + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
vllm/worker/model_runner.py
+133 -35
@@ -1,5 +1,5 @@
1 import time
2 -from typing import Dict, List, Tuple, Union
2 +from typing import Dict, List, Optional, Tuple, Union
3
4 import numpy as np
5 import torch
@@ -8,6 +8,8 @@ import torch.nn as nn
8 from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
9 from vllm.logger import init_logger
10 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11 +from vllm.model_executor.parallel_utils.communication_op import (
12 + broadcast, broadcast_object_list)
13 from vllm.sampling_params import SamplingParams, SamplingType
14 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15 from vllm.utils import in_wsl
@@ -28,10 +30,12 @@ class ModelRunner:
30 model_config: ModelConfig,
31 parallel_config: ParallelConfig,
32 scheduler_config: SchedulerConfig,
33 + is_driver_worker: bool = False,
34 ):
35 self.model_config = model_config
36 self.parallel_config = parallel_config
37 self.scheduler_config = scheduler_config
38 + self.is_driver_worker = is_driver_worker
39
40 # model_config can be None in tests/samplers/test_sampler.py.
41 # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
@@ -70,7 +74,7 @@ class ModelRunner:
74 def _prepare_prompt(
75 self,
76 seq_group_metadata_list: List[SequenceGroupMetadata],
73 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
77 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int]]:
78 assert len(seq_group_metadata_list) > 0
79 input_tokens: List[List[int]] = []
80 input_positions: List[List[int]] = []
@@ -135,14 +139,14 @@ class ModelRunner:
139 dtype=torch.long)
140
141 input_metadata = InputMetadata(
138 - prompt_lens=prompt_lens,
142 + is_prompt=True,
143 slot_mapping=slot_mapping,
144 max_context_len=None,
145 context_lens=None,
146 block_tables=None,
147 use_cuda_graph=False,
148 )
145 - return input_tokens, input_positions, input_metadata
149 + return input_tokens, input_positions, input_metadata, prompt_lens
150
151 def _prepare_decode(
152 self,
@@ -203,32 +207,24 @@ class ModelRunner:
207 block_tables.append([])
208 batch_size = graph_batch_size
209
206 - # When using CUDA graph, we don't need to make the tensors on the GPU
207 - # because they will be eventually copied to the designated GPU buffer.
208 - device = "cpu" if use_captured_graph else "cuda"
209 - pin_memory = use_captured_graph and not self.in_wsl
210 input_tokens = _make_tensor_with_pad(input_tokens,
211 max_len=1,
212 pad=0,
213 dtype=torch.long,
214 - device=device,
215 - pin_memory=pin_memory)
214 + device="cuda")
215 input_positions = _make_tensor_with_pad(input_positions,
216 max_len=1,
217 pad=0,
218 dtype=torch.long,
220 - device=device,
221 - pin_memory=pin_memory)
219 + device="cuda")
220 slot_mapping = _make_tensor_with_pad(slot_mapping,
221 max_len=1,
222 pad=_PAD_SLOT_ID,
223 dtype=torch.long,
226 - device=device,
227 - pin_memory=pin_memory)
224 + device="cuda")
225 context_lens = torch.tensor(context_lens,
226 dtype=torch.int,
230 - device=device,
231 - pin_memory=pin_memory)
227 + device="cuda")
228
229 if use_captured_graph:
230 # The shape of graph_block_tables is
@@ -237,17 +233,18 @@ class ModelRunner:
233 for i, block_table in enumerate(block_tables):
234 if block_table:
235 input_block_tables[i, :len(block_table)] = block_table
240 - block_tables = torch.tensor(input_block_tables, device=device)
236 + block_tables = torch.tensor(input_block_tables, device="cuda")
237 else:
238 block_tables = _make_tensor_with_pad(
239 block_tables,
240 max_len=max_context_len,
241 pad=0,
242 dtype=torch.int,
243 + device="cuda",
244 )
245
246 input_metadata = InputMetadata(
250 - prompt_lens=[],
247 + is_prompt=False,
248 slot_mapping=slot_mapping,
249 max_context_len=max_context_len,
250 context_lens=context_lens,
@@ -326,23 +323,127 @@ class ModelRunner:
323 )
324 return sampling_metadata
325
326 + def prepare_input_tensors(
327 + self,
328 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
329 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata]:
330 + if self.is_driver_worker:
331 + # NOTE: We assume that all sequences in the group are all prompts or
332 + # all decodes.
333 + is_prompt = seq_group_metadata_list[0].is_prompt
334 + # Prepare input tensors.
335 + if is_prompt:
336 + (input_tokens, input_positions, input_metadata,
337 + prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
338 + else:
339 + (input_tokens, input_positions, input_metadata
340 + ) = self._prepare_decode(seq_group_metadata_list)
341 + prompt_lens = []
342 + sampling_metadata = self._prepare_sample(seq_group_metadata_list,
343 + prompt_lens)
344 +
345 + def get_size_or_none(x: Optional[torch.Tensor]):
346 + return x.size() if x is not None else None
347 +
348 + # Broadcast the input data. For input tensors, we first broadcast
349 + # its shape and then broadcast the tensor to avoid high
350 + # serialization cost.
351 + py_data = {
352 + "input_tokens_size":
353 + input_tokens.size(),
354 + "input_positions_size":
355 + input_positions.size(),
356 + "is_prompt":
357 + input_metadata.is_prompt,
358 + "slot_mapping_size":
359 + get_size_or_none(input_metadata.slot_mapping),
360 + "max_context_len":
361 + input_metadata.max_context_len,
362 + "context_lens_size":
363 + get_size_or_none(input_metadata.context_lens),
364 + "block_tables_size":
365 + get_size_or_none(input_metadata.block_tables),
366 + "use_cuda_graph":
367 + input_metadata.use_cuda_graph,
368 + "selected_token_indices_size":
369 + sampling_metadata.selected_token_indices.size(),
370 + }
371 + broadcast_object_list([py_data], src=0)
372 + # TODO(zhuohan): Combine the broadcasts or set async_op=True.
373 + broadcast(input_tokens, src=0)
374 + broadcast(input_positions, src=0)
375 + if input_metadata.slot_mapping is not None:
376 + broadcast(input_metadata.slot_mapping, src=0)
377 + if input_metadata.context_lens is not None:
378 + broadcast(input_metadata.context_lens, src=0)
379 + if input_metadata.block_tables is not None:
380 + broadcast(input_metadata.block_tables, src=0)
381 + broadcast(sampling_metadata.selected_token_indices, src=0)
382 + else:
383 + receving_list = [None]
384 + broadcast_object_list(receving_list, src=0)
385 + py_data = receving_list[0]
386 + input_tokens = torch.empty(*py_data["input_tokens_size"],
387 + dtype=torch.long,
388 + device="cuda")
389 + broadcast(input_tokens, src=0)
390 + input_positions = torch.empty(*py_data["input_positions_size"],
391 + dtype=torch.long,
392 + device="cuda")
393 + broadcast(input_positions, src=0)
394 + if py_data["slot_mapping_size"] is not None:
395 + slot_mapping = torch.empty(*py_data["slot_mapping_size"],
396 + dtype=torch.long,
397 + device="cuda")
398 + broadcast(slot_mapping, src=0)
399 + else:
400 + slot_mapping = None
401 + if py_data["context_lens_size"] is not None:
402 + context_lens = torch.empty(*py_data["context_lens_size"],
403 + dtype=torch.int,
404 + device="cuda")
405 + broadcast(context_lens, src=0)
406 + else:
407 + context_lens = None
408 + if py_data["block_tables_size"] is not None:
409 + block_tables = torch.empty(*py_data["block_tables_size"],
410 + dtype=torch.int,
411 + device="cuda")
412 + broadcast(block_tables, src=0)
413 + else:
414 + block_tables = None
415 + selected_token_indices = torch.empty(
416 + *py_data["selected_token_indices_size"],
417 + dtype=torch.long,
418 + device="cuda")
419 + broadcast(selected_token_indices, src=0)
420 + input_metadata = InputMetadata(
421 + is_prompt=py_data["is_prompt"],
422 + slot_mapping=slot_mapping,
423 + max_context_len=py_data["max_context_len"],
424 + context_lens=context_lens,
425 + block_tables=block_tables,
426 + use_cuda_graph=py_data["use_cuda_graph"],
427 + )
428 + sampling_metadata = SamplingMetadata(
429 + seq_groups=None,
430 + seq_data=None,
431 + prompt_lens=None,
432 + selected_token_indices=selected_token_indices,
433 + categorized_sample_indices=None,
434 + perform_sampling=False,
435 + )
436 +
437 + return input_tokens, input_positions, input_metadata, sampling_metadata
438 +
439 @torch.inference_mode()
440 def execute_model(
441 self,
332 - seq_group_metadata_list: List[SequenceGroupMetadata],
442 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
443 kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
334 - ) -> SamplerOutput:
335 - # NOTE: We assume that all sequences in the group are all prompts or
336 - # all decodes.
337 - is_prompt = seq_group_metadata_list[0].is_prompt
338 - # Prepare input tensors.
339 - if is_prompt:
340 - inputs = self._prepare_prompt(seq_group_metadata_list)
341 - input_tokens, input_positions, input_metadata = inputs
342 - else:
343 - inputs = self._prepare_decode(seq_group_metadata_list)
344 - input_tokens, input_positions, input_metadata = inputs
345 -
444 + ) -> Optional[SamplerOutput]:
445 + input_tokens, input_positions, input_metadata, sampling_metadata = (
446 + self.prepare_input_tensors(seq_group_metadata_list))
447 # Execute the model.
448 if input_metadata.use_cuda_graph:
449 graph_batch_size = input_tokens.shape[0]
@@ -356,9 +457,6 @@ class ModelRunner:
457 input_metadata=input_metadata,
458 )
459
359 - sampling_metadata = self._prepare_sample(seq_group_metadata_list,
360 - input_metadata.prompt_lens)
361 -
460 # Sample the next token.
461 output = self.model.sample(
462 hidden_states=hidden_states,
@@ -424,7 +522,7 @@ class ModelRunner:
522 for batch_size in reversed(_BATCH_SIZES_TO_CAPTURE):
523 # Create dummy input_metadata.
524 input_metadata = InputMetadata(
427 - prompt_lens=[],
525 + is_prompt=False,
526 slot_mapping=slot_mapping[:batch_size],
527 max_context_len=self.max_context_len_to_capture,
528 context_lens=context_lens[:batch_size],
vllm/worker/worker.py
+45 -15
@@ -8,6 +8,8 @@ import torch.distributed
8 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig)
10 from vllm.model_executor import set_random_seed
11 +from vllm.model_executor.parallel_utils.communication_op import (
12 + broadcast_object_list)
13 from vllm.model_executor.parallel_utils.parallel_state import (
14 initialize_model_parallel)
15 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -28,17 +30,23 @@ class Worker:
30 model_config: ModelConfig,
31 parallel_config: ParallelConfig,
32 scheduler_config: SchedulerConfig,
31 - rank: Optional[int] = None,
32 - distributed_init_method: Optional[str] = None,
33 + local_rank: int,
34 + rank: int,
35 + distributed_init_method: str,
36 + is_driver_worker: bool = False,
37 ) -> None:
38 self.model_config = model_config
39 self.parallel_config = parallel_config
40 self.scheduler_config = scheduler_config
41 + self.local_rank = local_rank
42 self.rank = rank
43 self.distributed_init_method = distributed_init_method
44 + self.is_driver_worker = is_driver_worker
45 + if self.is_driver_worker:
46 + assert self.rank == 0, "The driver worker must have rank 0."
47
48 self.model_runner = ModelRunner(model_config, parallel_config,
41 - scheduler_config)
49 + scheduler_config, is_driver_worker)
50 # Uninitialized cache engine. Will be initialized by
51 # self.init_cache_engine().
52 self.cache_config = None
@@ -57,13 +65,7 @@ class Worker:
65
66 # This env var set by Ray causes exceptions with graph building.
67 os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
60 - # Env vars will be set by Ray.
61 - self.rank = self.rank if self.rank is not None else int(
62 - os.getenv("RANK", "-1"))
63 - local_rank = int(os.getenv("LOCAL_RANK", "0"))
64 - self.device = torch.device(f"cuda:{local_rank}")
65 - if self.rank < 0:
66 - raise ValueError("Invalid or unspecified rank.")
68 + self.device = torch.device(f"cuda:{self.local_rank}")
69 torch.cuda.set_device(self.device)
70
71 _check_if_gpu_supports_dtype(self.model_config.dtype)
@@ -125,14 +127,12 @@ class Worker:
127 # the model initialization and profiling.
128 set_random_seed(self.model_config.seed)
129
128 - @torch.inference_mode()
129 - def execute_model(
130 + def cache_swap(
131 self,
131 - seq_group_metadata_list: List[SequenceGroupMetadata],
132 blocks_to_swap_in: Dict[int, int],
133 blocks_to_swap_out: Dict[int, int],
134 blocks_to_copy: Dict[int, List[int]],
135 - ) -> SamplerOutput:
135 + ) -> None:
136 # Issue cache operations.
137 issued_cache_op = False
138 if blocks_to_swap_in:
@@ -152,8 +152,38 @@ class Worker:
152 if cache_events is not None:
153 for event in cache_events:
154 event.wait()
155 +
156 + @torch.inference_mode()
157 + def execute_model(
158 + self,
159 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
160 + blocks_to_swap_in: Optional[Dict[int, int]] = None,
161 + blocks_to_swap_out: Optional[Dict[int, int]] = None,
162 + blocks_to_copy: Optional[Dict[int, List[int]]] = None,
163 + ) -> Optional[SamplerOutput]:
164 + if self.is_driver_worker:
165 + assert seq_group_metadata_list is not None
166 + num_seq_groups = len(seq_group_metadata_list)
167 + assert blocks_to_swap_in is not None
168 + assert blocks_to_swap_out is not None
169 + assert blocks_to_copy is not None
170 + block_swapping_info = [
171 + blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy
172 + ]
173 + broadcast_object_list([num_seq_groups] + block_swapping_info,
174 + src=0)
175 + else:
176 + # num_seq_groups, blocks_to_swap_in, blocks_to_swap_out,
177 + # blocks_to_copy (4 elements)
178 + recv_data = [None] * 4
179 + broadcast_object_list(recv_data, src=0)
180 + num_seq_groups = recv_data[0]
181 + block_swapping_info = recv_data[1:]
182 +
183 + self.cache_swap(*block_swapping_info)
184 +
185 # If there is no input, we don't need to execute the model.
156 - if not seq_group_metadata_list:
186 + if num_seq_groups == 0:
187 return {}
188
189 output = self.model_runner.execute_model(seq_group_metadata_list,