A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Use NCCL instead of ray for control-plane communication to remove serialization overhead (#2221)
Zhuohan Li committed
Jan 4, 2024 at 03:30 UTC
fd4ea8ef5c17a8b991107402a414f6ed355d854d
34 files changed
+519
-257
docs/source/models/adding_model.rst
+3
-4
@@ -58,11 +58,10 @@ Next, you need to rewrite the :code:`forward` methods of your model by following
58
+ positions: torch.Tensor,
59
+ kv_caches: List[KVCache],
60
+ input_metadata: InputMetadata,
61
- + cache_events: Optional[List[torch.cuda.Event]],
62
- +) -> SamplerOutput:
61
+ +) -> Optional[SamplerOutput]:
62
64
-3. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
65
-4. Replace the attention operation with either :code:`PagedAttention`, :code:`PagedAttentionWithRoPE`, or :code:`PagedAttentionWithALiBi` depending on the model's architecture.
63
+1. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
64
+2. Replace the attention operation with either :code:`PagedAttention`, :code:`PagedAttentionWithRoPE`, or :code:`PagedAttentionWithALiBi` depending on the model's architecture.
65
66
.. note::
67
Currently, vLLM supports the basic multi-head attention mechanism and its variant with rotary positional embeddings.
requirements-rocm.txt
-2
@@ -3,8 +3,6 @@ typing-extensions>=4.8.0
3
starlette
4
psutil
5
ray >= 2.5.1
6
-pandas # Required for Ray data.
7
-pyarrow # Required for Ray data.
6
sentencepiece # Required for LLaMA tokenizer.
7
numpy
8
tokenizers>=0.15.0
requirements.txt
-2
@@ -1,8 +1,6 @@
1
ninja # For faster builds.
2
psutil
3
ray >= 2.5.1
4
-pandas # Required for Ray data.
5
-pyarrow # Required for Ray data.
4
sentencepiece # Required for LLaMA tokenizer.
5
numpy
6
torch == 2.1.2
tests/async_engine/test_api_server.py
+9
-4
@@ -8,11 +8,11 @@ import pytest
8
import requests
9
10
11
-def _query_server(prompt: str) -> dict:
11
+def _query_server(prompt: str, max_tokens: int = 5) -> dict:
12
response = requests.post("http://localhost:8000/generate",
13
json={
14
"prompt": prompt,
15
- "max_tokens": 100,
15
+ "max_tokens": max_tokens,
16
"temperature": 0,
17
"ignore_eos": True
18
})
@@ -20,6 +20,10 @@ def _query_server(prompt: str) -> dict:
20
return response.json()
21
22
23
+def _query_server_long(prompt: str) -> dict:
24
+ return _query_server(prompt, max_tokens=500)
25
+
26
+
27
@pytest.fixture
28
def api_server():
29
script_path = Path(__file__).parent.joinpath(
@@ -68,10 +72,11 @@ def test_api_server(api_server):
72
for result in pool.map(_query_server, prompts):
73
assert result
74
75
+ with Pool(32) as pool:
76
# Cancel requests
77
prompts = ["canceled requests"] * 100
73
- pool.map_async(_query_server, prompts)
74
- time.sleep(0.001)
78
+ pool.map_async(_query_server_long, prompts)
79
+ time.sleep(0.01)
80
pool.terminate()
81
pool.join()
82
tests/kernels/test_cache.py
+12
-12
@@ -49,12 +49,13 @@ def test_copy_blocks(
49
src_blocks = random.sample(range(num_blocks), num_mappings)
50
remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
51
dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)
52
- block_mapping = {}
52
+ copy_src = []
53
+ copy_dst = []
54
for i in range(num_mappings):
54
- src = src_blocks[i]
55
- dst1 = dst_blocks[2 * i]
56
- dst2 = dst_blocks[2 * i + 1]
57
- block_mapping[src] = [dst1, dst2]
55
+ copy_src.append(src_blocks[i])
56
+ copy_dst.append(dst_blocks[2 * i])
57
+ copy_src.append(src_blocks[i])
58
+ copy_dst.append(dst_blocks[2 * i + 1])
59
60
# Create the KV caches.
61
key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
@@ -66,15 +67,14 @@ def test_copy_blocks(
67
cloned_value_caches = [value_cache.clone() for value_cache in value_caches]
68
69
# Call the copy blocks kernel.
69
- cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
70
+ cache_ops.copy_blocks(key_caches, value_caches, copy_src, copy_dst)
71
72
# Run the reference implementation.
72
- for src, dsts in block_mapping.items():
73
- for dst in dsts:
74
- for cloned_key_cache in cloned_key_caches:
75
- cloned_key_cache[dst].copy_(cloned_key_cache[src])
76
- for cloned_value_cache in cloned_value_caches:
77
- cloned_value_cache[dst].copy_(cloned_value_cache[src])
73
+ for src, dst in zip(copy_src, copy_dst):
74
+ for cloned_key_cache in cloned_key_caches:
75
+ cloned_key_cache[dst].copy_(cloned_key_cache[src])
76
+ for cloned_value_cache in cloned_value_caches:
77
+ cloned_value_cache[dst].copy_(cloned_value_cache[src])
78
79
# Compare the results.
80
for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
tests/worker/test_model_runner.py
+3
-2
@@ -33,8 +33,9 @@ def test_prepare_prompt():
33
expected_selected_token_indices.append(selected_token_start_idx +
34
prompt_len - 1)
35
selected_token_start_idx += max_seq_len
36
- input_tokens, input_positions, _ = model_runner._prepare_prompt(
37
- seq_group_metadata_list)
36
+ input_tokens, input_positions, _, return_prompt_lens = (
37
+ model_runner._prepare_prompt(seq_group_metadata_list))
38
+ assert return_prompt_lens == prompt_lens
39
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
40
prompt_lens)
41
assert input_tokens.shape == (batch_size, max_seq_len)
vllm/engine/async_llm_engine.py
+33
-28
@@ -185,14 +185,21 @@ class _AsyncLLMEngine(LLMEngine):
185
"""
186
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
187
188
- # Execute the model.
189
- output = (await self._run_workers_async(
190
- "execute_model",
191
- seq_group_metadata_list=seq_group_metadata_list,
192
- blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
193
- blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
194
- blocks_to_copy=scheduler_outputs.blocks_to_copy,
195
- )) if not scheduler_outputs.is_empty() else []
188
+ if not scheduler_outputs.is_empty():
189
+ # Execute the model.
190
+ all_outputs = await self._run_workers_async(
191
+ "execute_model",
192
+ driver_kwargs={
193
+ "seq_group_metadata_list": seq_group_metadata_list,
194
+ "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
195
+ "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
196
+ "blocks_to_copy": scheduler_outputs.blocks_to_copy,
197
+ })
198
+
199
+ # Only the driver worker returns the sampling results.
200
+ output = all_outputs[0]
201
+ else:
202
+ output = []
203
204
return self._process_model_outputs(output, scheduler_outputs)
205
@@ -200,30 +207,29 @@ class _AsyncLLMEngine(LLMEngine):
207
self,
208
method: str,
209
*args,
203
- get_all_outputs: bool = False,
210
+ driver_args: Optional[List[Any]] = None,
211
+ driver_kwargs: Optional[Dict[str, Any]] = None,
212
**kwargs,
213
) -> Any:
214
"""Runs the given method on all workers."""
215
coros = []
208
- for worker in self.workers:
209
- if self.parallel_config.worker_use_ray:
210
- coros.append(
211
- worker.execute_method.remote(method, *args, **kwargs))
212
- else:
213
- executor = getattr(worker, method)
214
- coros.append(asyncio.get_event_loop().run_in_executor(
215
- None, partial(executor, *args, **kwargs)))
216
217
- all_outputs = await asyncio.gather(*coros)
217
+ if driver_args is None:
218
+ driver_args = args
219
+ if driver_kwargs is None:
220
+ driver_kwargs = kwargs
221
219
- if get_all_outputs:
220
- return all_outputs
222
+ # Run the driver worker asynchronously.
223
+ driver_executor = getattr(self.driver_worker, method)
224
+ coros.append(asyncio.get_event_loop().run_in_executor(
225
+ None, partial(driver_executor, *driver_args, **driver_kwargs)))
226
222
- # Make sure all workers have the same results.
223
- output = all_outputs[0]
224
- for other_output in all_outputs[1:]:
225
- assert output == other_output
226
- return output
227
+ # Run the ray workers asynchronously.
228
+ for worker in self.workers:
229
+ coros.append(worker.execute_method.remote(method, *args, **kwargs))
230
+
231
+ all_outputs = await asyncio.gather(*coros)
232
+ return all_outputs
233
234
235
class AsyncLLMEngine:
@@ -488,13 +494,12 @@ class AsyncLLMEngine:
494
engine_configs = engine_args.create_engine_configs()
495
parallel_config = engine_configs[2]
496
# Initialize the cluster.
491
- distributed_init_method, placement_group = initialize_cluster(
492
- parallel_config, engine_args.engine_use_ray)
497
+ placement_group = initialize_cluster(parallel_config,
498
+ engine_args.engine_use_ray)
499
# Create the async LLM engine.
500
engine = cls(parallel_config.worker_use_ray,
501
engine_args.engine_use_ray,
502
*engine_configs,
497
- distributed_init_method,
503
placement_group,
504
log_requests=not engine_args.disable_log_requests,
505
log_stats=not engine_args.disable_log_stats,
vllm/engine/llm_engine.py
+138
-100
@@ -1,8 +1,9 @@
1
import copy
2
+from collections import defaultdict
3
import os
4
import time
4
-from functools import partial
5
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union
5
+from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
6
+ Union)
7
8
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig)
@@ -17,10 +18,9 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
18
SequenceGroupOutput, SequenceOutput, SequenceStatus)
19
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
20
get_tokenizer)
20
-from vllm.utils import Counter
21
+from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port
22
23
if ray:
23
- from ray.air.util.torch_dist import init_torch_dist_process_group
24
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
25
26
if TYPE_CHECKING:
@@ -53,8 +53,6 @@ class LLMEngine:
53
management.
54
parallel_config: The configuration related to distributed execution.
55
scheduler_config: The configuration related to the request scheduler.
56
- distributed_init_method: The initialization method for distributed
57
- execution. See `torch.distributed.init_process_group` for details.
56
placement_group: Ray placement group for distributed execution.
57
Required for distributed execution.
58
log_stats: Whether to log statistics.
@@ -66,7 +64,6 @@ class LLMEngine:
64
cache_config: CacheConfig,
65
parallel_config: ParallelConfig,
66
scheduler_config: SchedulerConfig,
69
- distributed_init_method: str,
67
placement_group: Optional["PlacementGroup"],
68
log_stats: bool,
69
) -> None:
@@ -111,7 +108,7 @@ class LLMEngine:
108
os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
109
self._init_workers_ray(placement_group)
110
else:
114
- self._init_workers(distributed_init_method)
111
+ self._init_workers()
112
113
# Profile the memory usage and initialize the cache.
114
self._init_cache()
@@ -126,7 +123,7 @@ class LLMEngine:
123
# List of (timestamp, num_tokens)
124
self.num_generation_tokens: List[Tuple[float, int]] = []
125
129
- def _init_workers(self, distributed_init_method: str):
126
+ def _init_workers(self):
127
# Lazy import the Worker to avoid importing torch.cuda/xformers
128
# before CUDA_VISIBLE_DEVICES is set in the Worker
129
from vllm.worker.worker import Worker
@@ -135,70 +132,122 @@ class LLMEngine:
132
"Ray is required if parallel_config.world_size > 1.")
133
134
self.workers: List[Worker] = []
138
- worker = Worker(
135
+ distributed_init_method = f"tcp://{get_ip()}:{get_open_port()}"
136
+ self.driver_worker = Worker(
137
self.model_config,
138
self.parallel_config,
139
self.scheduler_config,
142
- 0,
143
- distributed_init_method,
144
- )
145
- self.workers.append(worker)
146
- self._run_workers(
147
- "init_model",
148
- get_all_outputs=True,
149
- )
150
- self._run_workers(
151
- "load_model",
152
- get_all_outputs=True,
153
- max_concurrent_workers=self.parallel_config.
154
- max_parallel_loading_workers,
140
+ local_rank=0,
141
+ rank=0,
142
+ distributed_init_method=distributed_init_method,
143
+ is_driver_worker=True,
144
)
145
+ self._run_workers("init_model")
146
+ self._run_workers("load_model")
147
148
def _init_workers_ray(self, placement_group: "PlacementGroup",
149
**ray_remote_kwargs):
159
- # Lazy import the Worker to avoid importing torch.cuda/xformers
160
- # before CUDA_VISIBLE_DEVICES is set in the Worker
161
- from vllm.worker.worker import Worker
150
+ if self.parallel_config.tensor_parallel_size == 1:
151
+ num_gpus = self.cache_config.gpu_memory_utilization
152
+ else:
153
+ num_gpus = 1
154
163
- self.workers: List[Worker] = []
164
- for bundle in placement_group.bundle_specs:
155
+ self.driver_dummy_worker: RayWorkerVllm = None
156
+ self.workers: List[RayWorkerVllm] = []
157
+
158
+ driver_ip = get_ip()
159
+ for bundle_id, bundle in enumerate(placement_group.bundle_specs):
160
if not bundle.get("GPU", 0):
161
continue
167
- if self.parallel_config.tensor_parallel_size == 1:
168
- num_gpus = self.cache_config.gpu_memory_utilization
169
- else:
170
- num_gpus = 1
162
+ scheduling_strategy = PlacementGroupSchedulingStrategy(
163
+ placement_group=placement_group,
164
+ placement_group_capture_child_tasks=True,
165
+ placement_group_bundle_index=bundle_id,
166
+ )
167
worker = ray.remote(
168
num_cpus=0,
169
num_gpus=num_gpus,
174
- scheduling_strategy=PlacementGroupSchedulingStrategy(
175
- placement_group=placement_group,
176
- placement_group_capture_child_tasks=True),
170
+ scheduling_strategy=scheduling_strategy,
171
**ray_remote_kwargs,
172
)(RayWorkerVllm).remote(self.model_config.trust_remote_code)
179
- self.workers.append(worker)
173
+
174
+ worker_ip = ray.get(worker.get_node_ip.remote())
175
+ if worker_ip == driver_ip and self.driver_dummy_worker is None:
176
+ # If the worker is on the same node as the driver, we use it
177
+ # as the resource holder for the driver process.
178
+ self.driver_dummy_worker = worker
179
+ else:
180
+ self.workers.append(worker)
181
+
182
+ if self.driver_dummy_worker is None:
183
+ raise ValueError(
184
+ "Ray does not allocate any GPUs on the driver node. Consider "
185
+ "adjusting the Ray placement group or running the driver on a "
186
+ "GPU node.")
187
+
188
+ driver_node_id, driver_gpu_ids = ray.get(
189
+ self.driver_dummy_worker.get_node_and_gpu_ids.remote())
190
+ worker_node_and_gpu_ids = ray.get(
191
+ [worker.get_node_and_gpu_ids.remote() for worker in self.workers])
192
+
193
+ node_workers = defaultdict(list)
194
+ node_gpus = defaultdict(list)
195
+
196
+ node_workers[driver_node_id].append(0)
197
+ node_gpus[driver_node_id].extend(driver_gpu_ids)
198
+ for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids,
199
+ start=1):
200
+ node_workers[node_id].append(i)
201
+ node_gpus[node_id].extend(gpu_ids)
202
+ for node_id, gpu_ids in node_gpus.items():
203
+ node_gpus[node_id] = sorted(gpu_ids)
204
+
205
+ # Set CUDA_VISIBLE_DEVICES for the driver.
206
+ set_cuda_visible_devices(node_gpus[driver_node_id])
207
+ for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
208
+ worker.set_cuda_visible_devices.remote(node_gpus[node_id])
209
+
210
+ distributed_init_method = f"tcp://{driver_ip}:{get_open_port()}"
211
+
212
+ # Lazy import the Worker to avoid importing torch.cuda/xformers
213
+ # before CUDA_VISIBLE_DEVICES is set in the Worker
214
+ from vllm.worker.worker import Worker
215
216
# Initialize torch distributed process group for the workers.
182
- init_torch_dist_process_group(self.workers, backend="nccl")
217
model_config = copy.deepcopy(self.model_config)
218
parallel_config = copy.deepcopy(self.parallel_config)
219
scheduler_config = copy.deepcopy(self.scheduler_config)
186
- self._run_workers("init_worker",
187
- get_all_outputs=True,
188
- worker_init_fn=lambda: Worker(
189
- model_config,
190
- parallel_config,
191
- scheduler_config,
192
- None,
193
- None,
194
- ))
195
- self._run_workers(
196
- "init_model",
197
- get_all_outputs=True,
220
+
221
+ for rank, (worker, (node_id,
222
+ _)) in enumerate(zip(self.workers,
223
+ worker_node_and_gpu_ids),
224
+ start=1):
225
+ local_rank = node_workers[node_id].index(rank)
226
+ worker.init_worker.remote(
227
+ lambda rank=rank, local_rank=local_rank: Worker(
228
+ model_config,
229
+ parallel_config,
230
+ scheduler_config,
231
+ local_rank,
232
+ rank,
233
+ distributed_init_method,
234
+ ))
235
+
236
+ driver_rank = 0
237
+ driver_local_rank = node_workers[driver_node_id].index(driver_rank)
238
+ self.driver_worker = Worker(
239
+ model_config,
240
+ parallel_config,
241
+ scheduler_config,
242
+ driver_local_rank,
243
+ driver_rank,
244
+ distributed_init_method,
245
+ is_driver_worker=True,
246
)
247
+
248
+ self._run_workers("init_model")
249
self._run_workers(
250
"load_model",
201
- get_all_outputs=True,
251
max_concurrent_workers=self.parallel_config.
252
max_parallel_loading_workers,
253
)
@@ -212,7 +261,6 @@ class LLMEngine:
261
# Get the maximum number of blocks that can be allocated on GPU and CPU.
262
num_blocks = self._run_workers(
263
"profile_num_available_blocks",
215
- get_all_outputs=True,
264
block_size=self.cache_config.block_size,
265
gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
266
cpu_swap_space=self.cache_config.swap_space_bytes,
@@ -256,11 +304,9 @@ class LLMEngine:
304
engine_configs = engine_args.create_engine_configs()
305
parallel_config = engine_configs[2]
306
# Initialize the cluster.
259
- distributed_init_method, placement_group = initialize_cluster(
260
- parallel_config)
307
+ placement_group = initialize_cluster(parallel_config)
308
# Create the LLM engine.
309
engine = cls(*engine_configs,
263
- distributed_init_method,
310
placement_group,
311
log_stats=not engine_args.disable_log_stats)
312
return engine
@@ -577,14 +623,21 @@ class LLMEngine:
623
"""
624
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
625
580
- # Execute the model.
581
- output = self._run_workers(
582
- "execute_model",
583
- seq_group_metadata_list=seq_group_metadata_list,
584
- blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
585
- blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
586
- blocks_to_copy=scheduler_outputs.blocks_to_copy,
587
- ) if not scheduler_outputs.is_empty() else []
626
+ if not scheduler_outputs.is_empty():
627
+ # Execute the model.
628
+ all_outputs = self._run_workers(
629
+ "execute_model",
630
+ driver_kwargs={
631
+ "seq_group_metadata_list": seq_group_metadata_list,
632
+ "blocks_to_swap_in": scheduler_outputs.blocks_to_swap_in,
633
+ "blocks_to_swap_out": scheduler_outputs.blocks_to_swap_out,
634
+ "blocks_to_copy": scheduler_outputs.blocks_to_copy,
635
+ })
636
+
637
+ # Only the driver worker returns the sampling results.
638
+ output = all_outputs[0]
639
+ else:
640
+ output = []
641
642
return self._process_model_outputs(output, scheduler_outputs)
643
@@ -712,53 +765,38 @@ class LLMEngine:
765
seq.status = SequenceStatus.FINISHED_STOPPED
766
return
767
715
- def _run_workers_in_batch(
716
- self,
717
- workers,
718
- method: str,
719
- *args,
720
- **kwargs,
721
- ):
722
- all_outputs = []
723
- for worker in workers:
724
- if self.parallel_config.worker_use_ray:
725
- executor = partial(worker.execute_method.remote, method)
726
- else:
727
- executor = getattr(worker, method)
728
-
729
- output = executor(*args, **kwargs)
730
- all_outputs.append(output)
731
- if self.parallel_config.worker_use_ray:
732
- all_outputs = ray.get(all_outputs)
733
- return all_outputs
734
-
768
def _run_workers(
769
self,
770
method: str,
771
*args,
739
- get_all_outputs: bool = False,
772
+ driver_args: Optional[List[Any]] = None,
773
+ driver_kwargs: Optional[Dict[str, Any]] = None,
774
max_concurrent_workers: Optional[int] = None,
775
**kwargs,
776
) -> Any:
777
"""Runs the given method on all workers."""
744
- all_outputs = []
778
+
779
if max_concurrent_workers:
746
- work_groups = [
747
- self.workers[i:i + max_concurrent_workers]
748
- for i in range(0, len(self.workers), max_concurrent_workers)
749
- ]
750
- else:
751
- work_groups = [self.workers]
780
+ raise NotImplementedError(
781
+ "max_concurrent_workers is not supported yet.")
782
+
783
+ # Start the ray workers first.
784
+ ray_worker_outputs = [
785
+ worker.execute_method.remote(method, *args, **kwargs)
786
+ for worker in self.workers
787
+ ]
788
+
789
+ if driver_args is None:
790
+ driver_args = args
791
+ if driver_kwargs is None:
792
+ driver_kwargs = kwargs
793
753
- for workers in work_groups:
754
- all_outputs.extend(
755
- self._run_workers_in_batch(workers, method, *args, **kwargs))
794
+ # Start the driver worker after all the ray workers.
795
+ driver_worker_output = getattr(self.driver_worker,
796
+ method)(*driver_args, **driver_kwargs)
797
757
- if get_all_outputs:
758
- return all_outputs
798
+ # Get the results of the ray workers.
799
+ if self.workers:
800
+ ray_worker_outputs = ray.get(ray_worker_outputs)
801
760
- # Make sure all workers have the same results.
761
- output = all_outputs[0]
762
- for other_output in all_outputs[1:]:
763
- assert output == other_output
764
- return output
802
+ return [driver_worker_output] + ray_worker_outputs
vllm/engine/ray_utils.py
+22
-15
@@ -1,16 +1,15 @@
1
-from typing import Optional, Tuple, TYPE_CHECKING
1
+from typing import Optional, List, Tuple, TYPE_CHECKING
2
3
from vllm.config import ParallelConfig
4
from vllm.logger import init_logger
5
-from vllm.utils import get_open_port, is_hip
5
+from vllm.utils import is_hip, set_cuda_visible_devices, get_ip
6
7
logger = init_logger(__name__)
8
9
try:
10
import ray
11
- from ray.air.util.torch_dist import TorchDistributedWorker
11
13
- class RayWorkerVllm(TorchDistributedWorker):
12
+ class RayWorkerVllm:
13
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
14
lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
15
@@ -30,12 +29,22 @@ try:
29
executor = getattr(self, method)
30
return executor(*args, **kwargs)
31
32
+ def get_node_ip(self) -> str:
33
+ return get_ip()
34
+
35
+ def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
36
+ node_id = ray.get_runtime_context().get_node_id()
37
+ gpu_ids = ray.get_gpu_ids()
38
+ return node_id, gpu_ids
39
+
40
+ def set_cuda_visible_devices(self, device_ids) -> None:
41
+ set_cuda_visible_devices(device_ids)
42
+
43
except ImportError as e:
44
logger.warning(f"Failed to import Ray with {e!r}. "
45
"For distributed inference, please install Ray with "
46
"`pip install ray pandas pyarrow`.")
47
ray = None
38
- TorchDistributedWorker = None
48
RayWorkerVllm = None
49
50
if TYPE_CHECKING:
@@ -75,13 +84,11 @@ def initialize_cluster(
84
ray.init(address=ray_address, ignore_reinit_error=True)
85
86
if not parallel_config.worker_use_ray:
78
- # Initialize cluster locally.
79
- port = get_open_port()
80
- # We need to setup the distributed init method to make sure
81
- # the distributed megatron code (e.g., get world size) works correctly.
82
- distributed_init_method = f"tcp://localhost:{port}"
83
- return distributed_init_method, None
87
+ assert parallel_config.world_size == 1, (
88
+ "Ray is required if parallel_config.world_size > 1.")
89
+ return None
90
91
+ # Create placement group for worker processes
92
current_placement_group = ray.util.get_current_placement_group()
93
if current_placement_group:
94
# We are in a placement group
@@ -106,12 +113,12 @@ def initialize_cluster(
113
"The number of required GPUs exceeds the total number of "
114
"available GPUs in the cluster.")
115
# Create a new placement group
109
- current_placement_group = ray.util.placement_group([{
110
- "GPU": 1
111
- }] * parallel_config.world_size)
116
+ placement_group_specs = ([{"GPU": 1}] * parallel_config.world_size)
117
+ current_placement_group = ray.util.placement_group(
118
+ placement_group_specs)
119
# Wait until PG is ready - this will block until all
120
# requested resources are available, and will timeout
121
# if they cannot be provisioned.
122
ray.get(current_placement_group.ready(), timeout=1800)
123
117
- return None, current_placement_group
124
+ return current_placement_group
vllm/model_executor/input_metadata.py
+4
-5
@@ -1,4 +1,4 @@
1
-from typing import List, Optional
1
+from typing import Optional
2
3
import torch
4
@@ -16,28 +16,27 @@ class InputMetadata:
16
17
def __init__(
18
self,
19
- prompt_lens: List[int],
19
+ is_prompt: bool,
20
slot_mapping: torch.Tensor,
21
max_context_len: Optional[int],
22
context_lens: Optional[torch.Tensor],
23
block_tables: Optional[torch.Tensor],
24
use_cuda_graph: bool,
25
) -> None:
26
- self.prompt_lens = prompt_lens
26
+ self.is_prompt = is_prompt
27
self.max_context_len = max_context_len
28
self.slot_mapping = slot_mapping
29
self.context_lens = context_lens
30
self.block_tables = block_tables
31
self.use_cuda_graph = use_cuda_graph
32
33
- self.is_prompt = len(prompt_lens) > 0
33
# Set during the execution of the first attention op.
34
# FIXME(woosuk): This is a hack.
35
self.attn_bias = None
36
37
def __repr__(self) -> str:
38
return ("InputMetadata("
40
- f"prompt_lens={self.prompt_lens}, "
39
+ f"is_prompt={self.is_prompt}, "
40
f"max_context_len={self.max_context_len}, "
41
f"slot_mapping={self.slot_mapping}, "
42
f"context_lens={self.context_lens}, "
vllm/model_executor/layers/sampler.py
+14
-5
@@ -5,7 +5,7 @@ import torch
5
import torch.nn as nn
6
7
from vllm.model_executor.parallel_utils.communication_op import (
8
- tensor_model_parallel_all_gather)
8
+ tensor_model_parallel_gather)
9
from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors
10
from vllm.sampling_params import SamplingParams, SamplingType
11
from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
@@ -37,7 +37,7 @@ class Sampler(nn.Module):
37
hidden_states: torch.Tensor,
38
sampling_metadata: SamplingMetadata,
39
embedding_bias: Optional[torch.Tensor] = None,
40
- ) -> SamplerOutput:
40
+ ) -> Optional[SamplerOutput]:
41
# Get the hidden states that we use for sampling.
42
hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)
43
@@ -45,6 +45,14 @@ class Sampler(nn.Module):
45
logits = _get_logits(hidden_states, embedding, embedding_bias,
46
self.vocab_size)
47
48
+ # Only perform sampling in the driver worker.
49
+ # Note: `_get_logits` is still distributed across TP workers because
50
+ # the `embedding` weight is distributed across TP workers.
51
+ # TODO(zhuohan): Change the get_logits part to a separate stage.
52
+ if not sampling_metadata.perform_sampling:
53
+ return None
54
+
55
+ assert logits is not None
56
_, vocab_size = logits.shape
57
58
# Apply logits processors (if any).
@@ -92,14 +100,15 @@ class Sampler(nn.Module):
100
101
def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
102
embedding_bias: Optional[torch.Tensor],
95
- vocab_size: int) -> torch.Tensor:
103
+ vocab_size: int) -> Optional[torch.Tensor]:
104
# Get the logits for the next tokens.
105
logits = torch.matmul(hidden_states, embedding.t())
106
if embedding_bias is not None:
107
logits += embedding_bias
100
- logits = tensor_model_parallel_all_gather(logits)
108
+ logits = tensor_model_parallel_gather(logits)
109
# Remove paddings in vocab (if any).
102
- logits = logits[:, :vocab_size]
110
+ if logits is not None:
111
+ logits = logits[:, :vocab_size]
112
return logits
113
114
vllm/model_executor/models/aquila.py
+1
-1
@@ -298,7 +298,7 @@ class AquilaForCausalLM(nn.Module):
298
self,
299
hidden_states: torch.Tensor,
300
sampling_metadata: SamplingMetadata,
301
- ) -> SamplerOutput:
301
+ ) -> Optional[SamplerOutput]:
302
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303
sampling_metadata)
304
return next_tokens
vllm/model_executor/models/baichuan.py
+1
-1
@@ -313,7 +313,7 @@ class BaiChuanBaseForCausalLM(nn.Module):
313
self,
314
hidden_states: torch.Tensor,
315
sampling_metadata: SamplingMetadata,
316
- ) -> SamplerOutput:
316
+ ) -> Optional[SamplerOutput]:
317
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
318
sampling_metadata)
319
return next_tokens
vllm/model_executor/models/bloom.py
+1
-1
@@ -290,7 +290,7 @@ class BloomForCausalLM(nn.Module):
290
self,
291
hidden_states: torch.Tensor,
292
sampling_metadata: SamplingMetadata,
293
- ) -> SamplerOutput:
293
+ ) -> Optional[SamplerOutput]:
294
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
295
sampling_metadata)
296
return next_tokens
vllm/model_executor/models/chatglm.py
+1
-1
@@ -349,7 +349,7 @@ class ChatGLMForCausalLM(nn.Module):
349
self,
350
hidden_states: torch.Tensor,
351
sampling_metadata: SamplingMetadata,
352
- ) -> SamplerOutput:
352
+ ) -> Optional[SamplerOutput]:
353
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
354
sampling_metadata)
355
return next_tokens
vllm/model_executor/models/falcon.py
+1
-1
@@ -394,7 +394,7 @@ class FalconForCausalLM(nn.Module):
394
self,
395
hidden_states: torch.Tensor,
396
sampling_metadata: SamplingMetadata,
397
- ) -> SamplerOutput:
397
+ ) -> Optional[SamplerOutput]:
398
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
399
sampling_metadata)
400
return next_tokens
vllm/model_executor/models/gpt2.py
+1
-1
@@ -235,7 +235,7 @@ class GPT2LMHeadModel(nn.Module):
235
self,
236
hidden_states: torch.Tensor,
237
sampling_metadata: SamplingMetadata,
238
- ) -> SamplerOutput:
238
+ ) -> Optional[SamplerOutput]:
239
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
240
sampling_metadata)
241
return next_tokens
vllm/model_executor/models/gpt_bigcode.py
+1
-1
@@ -254,7 +254,7 @@ class GPTBigCodeForCausalLM(nn.Module):
254
self,
255
hidden_states: torch.Tensor,
256
sampling_metadata: SamplingMetadata,
257
- ) -> SamplerOutput:
257
+ ) -> Optional[SamplerOutput]:
258
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
259
sampling_metadata)
260
return next_tokens
vllm/model_executor/models/gpt_j.py
+1
-1
@@ -240,7 +240,7 @@ class GPTJForCausalLM(nn.Module):
240
self,
241
hidden_states: torch.Tensor,
242
sampling_metadata: SamplingMetadata,
243
- ) -> SamplerOutput:
243
+ ) -> Optional[SamplerOutput]:
244
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
245
sampling_metadata, self.lm_head.bias)
246
return next_tokens
vllm/model_executor/models/gpt_neox.py
+1
-1
@@ -255,7 +255,7 @@ class GPTNeoXForCausalLM(nn.Module):
255
self,
256
hidden_states: torch.Tensor,
257
sampling_metadata: SamplingMetadata,
258
- ) -> SamplerOutput:
258
+ ) -> Optional[SamplerOutput]:
259
next_tokens = self.sampler(self.embed_out.weight, hidden_states,
260
sampling_metadata)
261
return next_tokens
vllm/model_executor/models/internlm.py
+1
-1
@@ -255,7 +255,7 @@ class InternLMForCausalLM(nn.Module):
255
self,
256
hidden_states: torch.Tensor,
257
sampling_metadata: SamplingMetadata,
258
- ) -> SamplerOutput:
258
+ ) -> Optional[SamplerOutput]:
259
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260
sampling_metadata)
261
return next_tokens
vllm/model_executor/models/llama.py
+1
-1
@@ -291,7 +291,7 @@ class LlamaForCausalLM(nn.Module):
291
self,
292
hidden_states: torch.Tensor,
293
sampling_metadata: SamplingMetadata,
294
- ) -> SamplerOutput:
294
+ ) -> Optional[SamplerOutput]:
295
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
296
sampling_metadata)
297
return next_tokens
vllm/model_executor/models/mistral.py
+1
-1
@@ -287,7 +287,7 @@ class MistralForCausalLM(nn.Module):
287
self,
288
hidden_states: torch.Tensor,
289
sampling_metadata: SamplingMetadata,
290
- ) -> SamplerOutput:
290
+ ) -> Optional[SamplerOutput]:
291
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
292
sampling_metadata)
293
return next_tokens
vllm/model_executor/models/mixtral.py
+2
-2
@@ -320,7 +320,7 @@ class MixtralModel(nn.Module):
320
positions: torch.Tensor,
321
kv_caches: List[KVCache],
322
input_metadata: InputMetadata,
323
- ) -> SamplerOutput:
323
+ ) -> torch.Tensor:
324
hidden_states = self.embed_tokens(input_ids)
325
residual = None
326
for i in range(len(self.layers)):
@@ -361,7 +361,7 @@ class MixtralForCausalLM(nn.Module):
361
self,
362
hidden_states: Optional[torch.Tensor],
363
sampling_metadata: SamplingMetadata,
364
- ) -> SamplerOutput:
364
+ ) -> Optional[SamplerOutput]:
365
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
366
sampling_metadata)
367
return next_tokens
vllm/model_executor/models/mpt.py
+1
-1
@@ -276,7 +276,7 @@ class MPTForCausalLM(nn.Module):
276
self,
277
hidden_states: torch.Tensor,
278
sampling_metadata: SamplingMetadata,
279
- ) -> SamplerOutput:
279
+ ) -> Optional[SamplerOutput]:
280
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
281
sampling_metadata)
282
return next_tokens
vllm/model_executor/models/opt.py
+1
-1
@@ -309,7 +309,7 @@ class OPTForCausalLM(nn.Module):
309
self,
310
hidden_states: torch.Tensor,
311
sampling_metadata: SamplingMetadata,
312
- ) -> SamplerOutput:
312
+ ) -> Optional[SamplerOutput]:
313
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
314
sampling_metadata)
315
return next_tokens
vllm/model_executor/models/phi_1_5.py
+1
-1
@@ -280,7 +280,7 @@ class PhiForCausalLM(nn.Module):
280
self,
281
hidden_states: torch.Tensor,
282
sampling_metadata: SamplingMetadata,
283
- ) -> SamplerOutput:
283
+ ) -> Optional[SamplerOutput]:
284
head = self.lm_head.linear
285
next_tokens = self.sampler(head.weight, hidden_states,
286
sampling_metadata, head.bias)
vllm/model_executor/models/qwen.py
+1
-1
@@ -247,7 +247,7 @@ class QWenLMHeadModel(nn.Module):
247
self,
248
hidden_states: torch.Tensor,
249
sampling_metadata: SamplingMetadata,
250
- ) -> SamplerOutput:
250
+ ) -> Optional[SamplerOutput]:
251
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
252
sampling_metadata)
253
return next_tokens
vllm/model_executor/models/yi.py
+1
-1
@@ -286,7 +286,7 @@ class YiForCausalLM(nn.Module):
286
self,
287
hidden_states: torch.Tensor,
288
sampling_metadata: SamplingMetadata,
289
- ) -> SamplerOutput:
289
+ ) -> Optional[SamplerOutput]:
290
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291
sampling_metadata)
292
return next_tokens
vllm/model_executor/parallel_utils/communication_op.py
+59
@@ -1,6 +1,7 @@
1
import torch
2
3
from vllm.model_executor.parallel_utils.parallel_state import (
4
+ get_tensor_model_parallel_rank,
5
get_tensor_model_parallel_world_size,
6
get_tensor_model_parallel_group,
7
)
@@ -45,3 +46,61 @@ def tensor_model_parallel_all_gather(input_, dim=-1):
46
(world_size * input_size[dim], ) +
47
input_size[dim + 1:])
48
return output_tensor
49
+
50
+
51
+def tensor_model_parallel_gather(input_, dst=0, dim=-1):
52
+ """Gather the input tensor across model parallel group.
53
+
54
+ NOTE: We assume that the input tensor is on the same device across
55
+ all the ranks.
56
+ """
57
+ world_size = get_tensor_model_parallel_world_size()
58
+ # Bypass the function if we are using only 1 GPU.
59
+ if world_size == 1:
60
+ return input_
61
+ assert -input_.dim() <= dim < input_.dim(), (
62
+ f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
63
+ if dim < 0:
64
+ # Convert negative dim to positive.
65
+ dim += input_.dim()
66
+ # Allocate output tensor.
67
+ if get_tensor_model_parallel_rank() == dst:
68
+ gather_list = [torch.empty_like(input_) for _ in range(world_size)]
69
+ else:
70
+ gather_list = None
71
+ # Gather.
72
+ torch.distributed.gather(input_,
73
+ gather_list,
74
+ dst=dst,
75
+ group=get_tensor_model_parallel_group())
76
+ if get_tensor_model_parallel_rank() == dst:
77
+ output_tensor = torch.cat(gather_list, dim=dim)
78
+ else:
79
+ output_tensor = None
80
+ return output_tensor
81
+
82
+
83
+def broadcast(input_, src=0):
84
+ """Broadcast the input tensor."""
85
+ world_size = torch.distributed.get_world_size()
86
+ assert 0 <= src < world_size, f"Invalid src rank ({src})"
87
+
88
+ # Bypass the function if we are using only 1 GPU.
89
+ if world_size == 1:
90
+ return input_
91
+ # Broadcast.
92
+ torch.distributed.broadcast(input_, src=src)
93
+ return input_
94
+
95
+
96
+def broadcast_object_list(obj_list, src=0):
97
+ """Broadcast the input object list."""
98
+ world_size = torch.distributed.get_world_size()
99
+ assert 0 <= src < world_size, f"Invalid src rank ({src})"
100
+
101
+ # Bypass the function if we are using only 1 GPU.
102
+ if world_size == 1:
103
+ return obj_list
104
+ # Broadcast.
105
+ torch.distributed.broadcast_object_list(obj_list, src=src)
106
+ return obj_list
vllm/model_executor/sampling_metadata.py
+14
-8
@@ -1,5 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import Dict, List, Tuple
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
@@ -18,24 +18,29 @@ class SamplingMetadata:
18
seq_data: Seq_id -> SequenceData.
19
prompt_lens: Lengths of prompts.
20
selected_token_indices: Token indices selected for sampling.
21
- categorized_sample_indices: SamplingType -> token indicies to sample.
21
+ categorized_sample_indices: SamplingType -> token indices to sample.
22
+ perform_sampling: Whether to perform sampling. This option is used to
23
+ make the sampling only happens in the driver worker, and disable
24
+ sampling in other worker processes.
25
"""
26
27
def __init__(
28
self,
26
- seq_groups: List[Tuple[List[int], SamplingParams]],
27
- seq_data: Dict[int, SequenceData],
28
- prompt_lens: List[int],
29
+ seq_groups: Optional[List[Tuple[List[int], SamplingParams]]],
30
+ seq_data: Optional[Dict[int, SequenceData]],
31
+ prompt_lens: Optional[List[int]],
32
selected_token_indices: torch.Tensor,
30
- categorized_sample_indices: Dict[SamplingType, torch.Tensor],
33
+ categorized_sample_indices: Optional[Dict[SamplingType, torch.Tensor]],
34
+ perform_sampling: bool = True,
35
) -> None:
36
self.seq_groups = seq_groups
37
self.seq_data = seq_data
38
self.prompt_lens = prompt_lens
39
self.selected_token_indices = selected_token_indices
40
self.categorized_sample_indices = categorized_sample_indices
41
+ self.perform_sampling = perform_sampling
42
38
- self.num_prompts = len(prompt_lens)
43
+ self.num_prompts = len(prompt_lens) if prompt_lens is not None else 0
44
45
def __repr__(self) -> str:
46
return (
@@ -44,7 +49,8 @@ class SamplingMetadata:
49
f"seq_data={self.seq_data}, "
50
f"prompt_lens={self.prompt_lens}, "
51
f"selected_token_indices={self.selected_token_indices}, "
47
- f"categorized_sample_indices={self.categorized_sample_indices})")
52
+ f"categorized_sample_indices={self.categorized_sample_indices}), "
53
+ f"perform_sampling={self.perform_sampling})")
54
55
56
@dataclass
vllm/utils.py
+11
-1
@@ -1,7 +1,9 @@
1
import enum
2
+import os
3
import socket
4
import uuid
5
from platform import uname
6
+from typing import List
7
8
import psutil
9
import torch
@@ -55,7 +57,15 @@ def in_wsl() -> bool:
57
return "microsoft" in " ".join(uname()).lower()
58
59
58
-def get_open_port():
60
+def get_ip() -> str:
61
+ return socket.gethostbyname(socket.gethostname())
62
+
63
+
64
+def get_open_port() -> int:
65
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
66
s.bind(("", 0))
67
return s.getsockname()[1]
68
+
69
+
70
+def set_cuda_visible_devices(device_ids: List[int]) -> None:
71
+ os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
vllm/worker/model_runner.py
+133
-35
@@ -1,5 +1,5 @@
1
import time
2
-from typing import Dict, List, Tuple, Union
2
+from typing import Dict, List, Optional, Tuple, Union
3
4
import numpy as np
5
import torch
@@ -8,6 +8,8 @@ import torch.nn as nn
8
from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
9
from vllm.logger import init_logger
10
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11
+from vllm.model_executor.parallel_utils.communication_op import (
12
+ broadcast, broadcast_object_list)
13
from vllm.sampling_params import SamplingParams, SamplingType
14
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
from vllm.utils import in_wsl
@@ -28,10 +30,12 @@ class ModelRunner:
30
model_config: ModelConfig,
31
parallel_config: ParallelConfig,
32
scheduler_config: SchedulerConfig,
33
+ is_driver_worker: bool = False,
34
):
35
self.model_config = model_config
36
self.parallel_config = parallel_config
37
self.scheduler_config = scheduler_config
38
+ self.is_driver_worker = is_driver_worker
39
40
# model_config can be None in tests/samplers/test_sampler.py.
41
# FIXME(woosuk): This is a hack to make the tests work. Refactor this.
@@ -70,7 +74,7 @@ class ModelRunner:
74
def _prepare_prompt(
75
self,
76
seq_group_metadata_list: List[SequenceGroupMetadata],
73
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
77
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int]]:
78
assert len(seq_group_metadata_list) > 0
79
input_tokens: List[List[int]] = []
80
input_positions: List[List[int]] = []
@@ -135,14 +139,14 @@ class ModelRunner:
139
dtype=torch.long)
140
141
input_metadata = InputMetadata(
138
- prompt_lens=prompt_lens,
142
+ is_prompt=True,
143
slot_mapping=slot_mapping,
144
max_context_len=None,
145
context_lens=None,
146
block_tables=None,
147
use_cuda_graph=False,
148
)
145
- return input_tokens, input_positions, input_metadata
149
+ return input_tokens, input_positions, input_metadata, prompt_lens
150
151
def _prepare_decode(
152
self,
@@ -203,32 +207,24 @@ class ModelRunner:
207
block_tables.append([])
208
batch_size = graph_batch_size
209
206
- # When using CUDA graph, we don't need to make the tensors on the GPU
207
- # because they will be eventually copied to the designated GPU buffer.
208
- device = "cpu" if use_captured_graph else "cuda"
209
- pin_memory = use_captured_graph and not self.in_wsl
210
input_tokens = _make_tensor_with_pad(input_tokens,
211
max_len=1,
212
pad=0,
213
dtype=torch.long,
214
- device=device,
215
- pin_memory=pin_memory)
214
+ device="cuda")
215
input_positions = _make_tensor_with_pad(input_positions,
216
max_len=1,
217
pad=0,
218
dtype=torch.long,
220
- device=device,
221
- pin_memory=pin_memory)
219
+ device="cuda")
220
slot_mapping = _make_tensor_with_pad(slot_mapping,
221
max_len=1,
222
pad=_PAD_SLOT_ID,
223
dtype=torch.long,
226
- device=device,
227
- pin_memory=pin_memory)
224
+ device="cuda")
225
context_lens = torch.tensor(context_lens,
226
dtype=torch.int,
230
- device=device,
231
- pin_memory=pin_memory)
227
+ device="cuda")
228
229
if use_captured_graph:
230
# The shape of graph_block_tables is
@@ -237,17 +233,18 @@ class ModelRunner:
233
for i, block_table in enumerate(block_tables):
234
if block_table:
235
input_block_tables[i, :len(block_table)] = block_table
240
- block_tables = torch.tensor(input_block_tables, device=device)
236
+ block_tables = torch.tensor(input_block_tables, device="cuda")
237
else:
238
block_tables = _make_tensor_with_pad(
239
block_tables,
240
max_len=max_context_len,
241
pad=0,
242
dtype=torch.int,
243
+ device="cuda",
244
)
245
246
input_metadata = InputMetadata(
250
- prompt_lens=[],
247
+ is_prompt=False,
248
slot_mapping=slot_mapping,
249
max_context_len=max_context_len,
250
context_lens=context_lens,
@@ -326,23 +323,127 @@ class ModelRunner:
323
)
324
return sampling_metadata
325
326
+ def prepare_input_tensors(
327
+ self,
328
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
329
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata]:
330
+ if self.is_driver_worker:
331
+ # NOTE: We assume that all sequences in the group are all prompts or
332
+ # all decodes.
333
+ is_prompt = seq_group_metadata_list[0].is_prompt
334
+ # Prepare input tensors.
335
+ if is_prompt:
336
+ (input_tokens, input_positions, input_metadata,
337
+ prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
338
+ else:
339
+ (input_tokens, input_positions, input_metadata
340
+ ) = self._prepare_decode(seq_group_metadata_list)
341
+ prompt_lens = []
342
+ sampling_metadata = self._prepare_sample(seq_group_metadata_list,
343
+ prompt_lens)
344
+
345
+ def get_size_or_none(x: Optional[torch.Tensor]):
346
+ return x.size() if x is not None else None
347
+
348
+ # Broadcast the input data. For input tensors, we first broadcast
349
+ # its shape and then broadcast the tensor to avoid high
350
+ # serialization cost.
351
+ py_data = {
352
+ "input_tokens_size":
353
+ input_tokens.size(),
354
+ "input_positions_size":
355
+ input_positions.size(),
356
+ "is_prompt":
357
+ input_metadata.is_prompt,
358
+ "slot_mapping_size":
359
+ get_size_or_none(input_metadata.slot_mapping),
360
+ "max_context_len":
361
+ input_metadata.max_context_len,
362
+ "context_lens_size":
363
+ get_size_or_none(input_metadata.context_lens),
364
+ "block_tables_size":
365
+ get_size_or_none(input_metadata.block_tables),
366
+ "use_cuda_graph":
367
+ input_metadata.use_cuda_graph,
368
+ "selected_token_indices_size":
369
+ sampling_metadata.selected_token_indices.size(),
370
+ }
371
+ broadcast_object_list([py_data], src=0)
372
+ # TODO(zhuohan): Combine the broadcasts or set async_op=True.
373
+ broadcast(input_tokens, src=0)
374
+ broadcast(input_positions, src=0)
375
+ if input_metadata.slot_mapping is not None:
376
+ broadcast(input_metadata.slot_mapping, src=0)
377
+ if input_metadata.context_lens is not None:
378
+ broadcast(input_metadata.context_lens, src=0)
379
+ if input_metadata.block_tables is not None:
380
+ broadcast(input_metadata.block_tables, src=0)
381
+ broadcast(sampling_metadata.selected_token_indices, src=0)
382
+ else:
383
+ receving_list = [None]
384
+ broadcast_object_list(receving_list, src=0)
385
+ py_data = receving_list[0]
386
+ input_tokens = torch.empty(*py_data["input_tokens_size"],
387
+ dtype=torch.long,
388
+ device="cuda")
389
+ broadcast(input_tokens, src=0)
390
+ input_positions = torch.empty(*py_data["input_positions_size"],
391
+ dtype=torch.long,
392
+ device="cuda")
393
+ broadcast(input_positions, src=0)
394
+ if py_data["slot_mapping_size"] is not None:
395
+ slot_mapping = torch.empty(*py_data["slot_mapping_size"],
396
+ dtype=torch.long,
397
+ device="cuda")
398
+ broadcast(slot_mapping, src=0)
399
+ else:
400
+ slot_mapping = None
401
+ if py_data["context_lens_size"] is not None:
402
+ context_lens = torch.empty(*py_data["context_lens_size"],
403
+ dtype=torch.int,
404
+ device="cuda")
405
+ broadcast(context_lens, src=0)
406
+ else:
407
+ context_lens = None
408
+ if py_data["block_tables_size"] is not None:
409
+ block_tables = torch.empty(*py_data["block_tables_size"],
410
+ dtype=torch.int,
411
+ device="cuda")
412
+ broadcast(block_tables, src=0)
413
+ else:
414
+ block_tables = None
415
+ selected_token_indices = torch.empty(
416
+ *py_data["selected_token_indices_size"],
417
+ dtype=torch.long,
418
+ device="cuda")
419
+ broadcast(selected_token_indices, src=0)
420
+ input_metadata = InputMetadata(
421
+ is_prompt=py_data["is_prompt"],
422
+ slot_mapping=slot_mapping,
423
+ max_context_len=py_data["max_context_len"],
424
+ context_lens=context_lens,
425
+ block_tables=block_tables,
426
+ use_cuda_graph=py_data["use_cuda_graph"],
427
+ )
428
+ sampling_metadata = SamplingMetadata(
429
+ seq_groups=None,
430
+ seq_data=None,
431
+ prompt_lens=None,
432
+ selected_token_indices=selected_token_indices,
433
+ categorized_sample_indices=None,
434
+ perform_sampling=False,
435
+ )
436
+
437
+ return input_tokens, input_positions, input_metadata, sampling_metadata
438
+
439
@torch.inference_mode()
440
def execute_model(
441
self,
332
- seq_group_metadata_list: List[SequenceGroupMetadata],
442
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
443
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
334
- ) -> SamplerOutput:
335
- # NOTE: We assume that all sequences in the group are all prompts or
336
- # all decodes.
337
- is_prompt = seq_group_metadata_list[0].is_prompt
338
- # Prepare input tensors.
339
- if is_prompt:
340
- inputs = self._prepare_prompt(seq_group_metadata_list)
341
- input_tokens, input_positions, input_metadata = inputs
342
- else:
343
- inputs = self._prepare_decode(seq_group_metadata_list)
344
- input_tokens, input_positions, input_metadata = inputs
345
-
444
+ ) -> Optional[SamplerOutput]:
445
+ input_tokens, input_positions, input_metadata, sampling_metadata = (
446
+ self.prepare_input_tensors(seq_group_metadata_list))
447
# Execute the model.
448
if input_metadata.use_cuda_graph:
449
graph_batch_size = input_tokens.shape[0]
@@ -356,9 +457,6 @@ class ModelRunner:
457
input_metadata=input_metadata,
458
)
459
359
- sampling_metadata = self._prepare_sample(seq_group_metadata_list,
360
- input_metadata.prompt_lens)
361
-
460
# Sample the next token.
461
output = self.model.sample(
462
hidden_states=hidden_states,
@@ -424,7 +522,7 @@ class ModelRunner:
522
for batch_size in reversed(_BATCH_SIZES_TO_CAPTURE):
523
# Create dummy input_metadata.
524
input_metadata = InputMetadata(
427
- prompt_lens=[],
525
+ is_prompt=False,
526
slot_mapping=slot_mapping[:batch_size],
527
max_context_len=self.max_context_len_to_capture,
528
context_lens=context_lens[:batch_size],
vllm/worker/worker.py
+45
-15
@@ -8,6 +8,8 @@ import torch.distributed
8
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig)
10
from vllm.model_executor import set_random_seed
11
+from vllm.model_executor.parallel_utils.communication_op import (
12
+ broadcast_object_list)
13
from vllm.model_executor.parallel_utils.parallel_state import (
14
initialize_model_parallel)
15
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -28,17 +30,23 @@ class Worker:
30
model_config: ModelConfig,
31
parallel_config: ParallelConfig,
32
scheduler_config: SchedulerConfig,
31
- rank: Optional[int] = None,
32
- distributed_init_method: Optional[str] = None,
33
+ local_rank: int,
34
+ rank: int,
35
+ distributed_init_method: str,
36
+ is_driver_worker: bool = False,
37
) -> None:
38
self.model_config = model_config
39
self.parallel_config = parallel_config
40
self.scheduler_config = scheduler_config
41
+ self.local_rank = local_rank
42
self.rank = rank
43
self.distributed_init_method = distributed_init_method
44
+ self.is_driver_worker = is_driver_worker
45
+ if self.is_driver_worker:
46
+ assert self.rank == 0, "The driver worker must have rank 0."
47
48
self.model_runner = ModelRunner(model_config, parallel_config,
41
- scheduler_config)
49
+ scheduler_config, is_driver_worker)
50
# Uninitialized cache engine. Will be initialized by
51
# self.init_cache_engine().
52
self.cache_config = None
@@ -57,13 +65,7 @@ class Worker:
65
66
# This env var set by Ray causes exceptions with graph building.
67
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
60
- # Env vars will be set by Ray.
61
- self.rank = self.rank if self.rank is not None else int(
62
- os.getenv("RANK", "-1"))
63
- local_rank = int(os.getenv("LOCAL_RANK", "0"))
64
- self.device = torch.device(f"cuda:{local_rank}")
65
- if self.rank < 0:
66
- raise ValueError("Invalid or unspecified rank.")
68
+ self.device = torch.device(f"cuda:{self.local_rank}")
69
torch.cuda.set_device(self.device)
70
71
_check_if_gpu_supports_dtype(self.model_config.dtype)
@@ -125,14 +127,12 @@ class Worker:
127
# the model initialization and profiling.
128
set_random_seed(self.model_config.seed)
129
128
- @torch.inference_mode()
129
- def execute_model(
130
+ def cache_swap(
131
self,
131
- seq_group_metadata_list: List[SequenceGroupMetadata],
132
blocks_to_swap_in: Dict[int, int],
133
blocks_to_swap_out: Dict[int, int],
134
blocks_to_copy: Dict[int, List[int]],
135
- ) -> SamplerOutput:
135
+ ) -> None:
136
# Issue cache operations.
137
issued_cache_op = False
138
if blocks_to_swap_in:
@@ -152,8 +152,38 @@ class Worker:
152
if cache_events is not None:
153
for event in cache_events:
154
event.wait()
155
+
156
+ @torch.inference_mode()
157
+ def execute_model(
158
+ self,
159
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
160
+ blocks_to_swap_in: Optional[Dict[int, int]] = None,
161
+ blocks_to_swap_out: Optional[Dict[int, int]] = None,
162
+ blocks_to_copy: Optional[Dict[int, List[int]]] = None,
163
+ ) -> Optional[SamplerOutput]:
164
+ if self.is_driver_worker:
165
+ assert seq_group_metadata_list is not None
166
+ num_seq_groups = len(seq_group_metadata_list)
167
+ assert blocks_to_swap_in is not None
168
+ assert blocks_to_swap_out is not None
169
+ assert blocks_to_copy is not None
170
+ block_swapping_info = [
171
+ blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy
172
+ ]
173
+ broadcast_object_list([num_seq_groups] + block_swapping_info,
174
+ src=0)
175
+ else:
176
+ # num_seq_groups, blocks_to_swap_in, blocks_to_swap_out,
177
+ # blocks_to_copy (4 elements)
178
+ recv_data = [None] * 4
179
+ broadcast_object_list(recv_data, src=0)
180
+ num_seq_groups = recv_data[0]
181
+ block_swapping_info = recv_data[1:]
182
+
183
+ self.cache_swap(*block_swapping_info)
184
+
185
# If there is no input, we don't need to execute the model.
156
- if not seq_group_metadata_list:
186
+ if num_seq_groups == 0:
187
return {}
188
189
output = self.model_runner.execute_model(seq_group_metadata_list,