A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
Support torchrun and SPMD-style offline inference (#12071)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 16, 2025 at 19:58 UTC
bf53e0c70b0fe17087914cc770fd801e0bf02137
14 files changed
+248
-30
.buildkite/test-pipeline.yaml
+1
@@ -463,6 +463,7 @@ steps:
463
- vllm/worker/worker.py
464
- vllm/worker/model_runner.py
465
commands:
466
+ - torchrun --nproc-per-node=2 distributed/test_torchrun_example.py
467
- pytest -v -s ./compile/test_basic_correctness.py
468
- pytest -v -s ./compile/test_wrapper.py
469
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
examples/offline_inference/torchrun_example.py
new
+64
@@ -0,0 +1,64 @@
1
+"""
2
+experimental support for tensor-parallel inference with torchrun,
3
+see https://github.com/vllm-project/vllm/issues/11400 for
4
+the motivation and use case for this example.
5
+run the script with `torchrun --nproc-per-node=2 torchrun_example.py`,
6
+the argument 2 should match the `tensor_parallel_size` below.
7
+see `tests/distributed/test_torchrun_example.py` for the unit test.
8
+"""
9
+
10
+from vllm import LLM, SamplingParams
11
+
12
+# Create prompts, the same across all ranks
13
+prompts = [
14
+ "Hello, my name is",
15
+ "The president of the United States is",
16
+ "The capital of France is",
17
+ "The future of AI is",
18
+]
19
+
20
+# Create sampling parameters, the same across all ranks
21
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
22
+
23
+# Use `distributed_executor_backend="external_launcher"` so that
24
+# this llm engine/instance only creates one worker.
25
+llm = LLM(
26
+ model="facebook/opt-125m",
27
+ tensor_parallel_size=2,
28
+ distributed_executor_backend="external_launcher",
29
+)
30
+
31
+outputs = llm.generate(prompts, sampling_params)
32
+
33
+# all ranks will have the same outputs
34
+for output in outputs:
35
+ prompt = output.prompt
36
+ generated_text = output.outputs[0].text
37
+ print(f"Prompt: {prompt!r}, "
38
+ f"Generated text: {generated_text!r}")
39
+"""
40
+Further tips:
41
+
42
+1. to communicate control messages across all ranks, use the cpu group,
43
+a PyTorch ProcessGroup with GLOO backend.
44
+
45
+```python
46
+from vllm.distributed.parallel_state import get_world_group
47
+cpu_group = get_world_group().cpu_group
48
+torch_rank = dist.get_rank(group=cpu_group)
49
+if torch_rank == 0:
50
+ # do something for rank 0, e.g. saving the results to disk.
51
+```
52
+
53
+2. to communicate data across all ranks, use the model's device group,
54
+a PyTorch ProcessGroup with NCCL backend.
55
+```python
56
+from vllm.distributed.parallel_state import get_world_group
57
+device_group = get_world_group().device_group
58
+```
59
+
60
+3. to access the model directly in every rank, use the following code:
61
+```python
62
+llm.llm_engine.model_executor.driver_worker.worker.model_runner.model
63
+```
64
+"""
tests/distributed/test_torchrun_example.py
new
+56
@@ -0,0 +1,56 @@
1
+# unit test for `examples/offline_inference/torchrun_example.py`
2
+
3
+import random
4
+
5
+import torch.distributed as dist
6
+
7
+from vllm import LLM, SamplingParams
8
+from vllm.distributed.parallel_state import get_world_group
9
+
10
+# Create prompts
11
+prompts = [
12
+ "Hello, my name is",
13
+ "The president of the United States is",
14
+ "The capital of France is",
15
+ "The future of AI is",
16
+]
17
+
18
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
19
+
20
+# set different `gpu_memory_utilization` and `swap_space` for different ranks,
21
+# to test if all ranks agree on the same kv cache configuration.
22
+llm = LLM(model="facebook/opt-125m",
23
+ tensor_parallel_size=2,
24
+ distributed_executor_backend="external_launcher",
25
+ gpu_memory_utilization=random.uniform(0.7, 0.9),
26
+ swap_space=random.randint(1, 4))
27
+
28
+outputs = llm.generate(prompts, sampling_params)
29
+
30
+cpu_group = get_world_group().cpu_group
31
+
32
+torch_rank = dist.get_rank(group=cpu_group)
33
+
34
+
35
+def test_consistent_across_ranks(obj):
36
+ if torch_rank == 0:
37
+ dist.broadcast_object_list([obj], src=0, group=cpu_group)
38
+ else:
39
+ container = [None]
40
+ dist.broadcast_object_list(container, src=0, group=cpu_group)
41
+ assert container[0] == obj
42
+
43
+
44
+test_consistent_across_ranks(
45
+ llm.llm_engine.vllm_config.cache_config.num_cpu_blocks)
46
+test_consistent_across_ranks(
47
+ llm.llm_engine.vllm_config.cache_config.num_gpu_blocks)
48
+
49
+# all ranks should have the same outputs
50
+for output in outputs:
51
+ prompt = output.prompt
52
+ generated_text = output.outputs[0].text
53
+ test_consistent_across_ranks(prompt)
54
+ test_consistent_across_ranks(generated_text)
55
+ print(f"Rank {torch_rank}, Prompt: {prompt!r}, "
56
+ f"Generated text: {generated_text!r}")
tests/engine/test_multiproc_workers.py
+1
-1
@@ -22,7 +22,7 @@ class DummyWorkerWrapper(WorkerWrapperBase):
22
# simulate error case
23
raise worker_input
24
25
- return self.rank, input
25
+ return self.rpc_rank, input
26
27
28
def _start_workers() -> Tuple[List[ProcessWorkerWrapper], WorkerMonitor]:
vllm/config.py
+4
-3
@@ -1338,14 +1338,15 @@ class ParallelConfig:
1338
from vllm.executor.executor_base import ExecutorBase
1339
from vllm.platforms import current_platform
1340
if self.distributed_executor_backend not in (
1341
- "ray", "mp", "uni", None) and not (isinstance(
1341
+ "ray", "mp", "uni",
1342
+ "external_launcher", None) and not (isinstance(
1343
self.distributed_executor_backend, type) and issubclass(
1344
self.distributed_executor_backend, ExecutorBase)):
1345
raise ValueError(
1346
"Unrecognized distributed executor backend "
1347
f"{self.distributed_executor_backend}. Supported "
1347
- "values are 'ray', 'mp' 'uni', or custom ExecutorBase"
1348
- " subclass.")
1348
+ "values are 'ray', 'mp' 'uni', 'external_launcher' or"
1349
+ " custom ExecutorBase subclass.")
1350
if self.use_ray:
1351
from vllm.executor import ray_utils
1352
ray_utils.assert_ray_available()
vllm/engine/arg_utils.py
+1
-1
@@ -388,7 +388,7 @@ class EngineArgs:
388
# Parallel arguments
389
parser.add_argument(
390
'--distributed-executor-backend',
391
- choices=['ray', 'mp'],
391
+ choices=['ray', 'mp', 'uni', 'external_launcher'],
392
default=EngineArgs.distributed_executor_backend,
393
help='Backend to use for distributed model '
394
'workers, either "ray" or "mp" (multiprocessing). If the product '
vllm/engine/llm_engine.py
+5
@@ -457,6 +457,11 @@ class LLMEngine:
457
# JAX-style, single-process, multi-device executor.
458
from vllm.executor.uniproc_executor import UniProcExecutor
459
executor_class = UniProcExecutor
460
+ elif distributed_executor_backend == "external_launcher":
461
+ # executor with external launcher
462
+ from vllm.executor.uniproc_executor import ( # noqa
463
+ ExecutorWithExternalLauncher)
464
+ executor_class = ExecutorWithExternalLauncher
465
else:
466
from vllm.executor.uniproc_executor import UniProcExecutor
467
executor_class = UniProcExecutor
vllm/executor/ray_distributed_executor.py
+3
-3
@@ -172,7 +172,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
172
scheduling_strategy=scheduling_strategy,
173
**ray_remote_kwargs,
174
)(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
175
- rank=rank)
175
+ rpc_rank=rank)
176
else:
177
worker = ray.remote(
178
num_cpus=0,
@@ -181,7 +181,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
181
scheduling_strategy=scheduling_strategy,
182
**ray_remote_kwargs,
183
)(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
184
- rank=rank)
184
+ rpc_rank=rank)
185
worker_metadata.append(
186
RayWorkerMetaData(worker=worker, created_rank=rank))
187
rank += 1
@@ -204,7 +204,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
204
# as the resource holder for the driver process.
205
self.driver_dummy_worker = worker
206
self.driver_worker = RayWorkerWrapper(
207
- vllm_config=self.vllm_config, rank=0)
207
+ vllm_config=self.vllm_config, rpc_rank=0)
208
worker_metadata.pop(i)
209
break
210
vllm/executor/uniproc_executor.py
+80
-1
@@ -1,5 +1,10 @@
1
+import os
2
from typing import Any, Dict, List, Optional, Tuple
3
4
+import torch
5
+import torch.distributed as dist
6
+
7
+import vllm.envs as envs
8
from vllm.executor.executor_base import ExecutorBase
9
from vllm.logger import init_logger
10
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
@@ -16,7 +21,7 @@ class UniProcExecutor(ExecutorBase):
21
"""Initialize the worker and load the model.
22
"""
23
self.driver_worker = WorkerWrapperBase(vllm_config=self.vllm_config,
19
- rank=0)
24
+ rpc_rank=0)
25
distributed_init_method = get_distributed_init_method(
26
get_ip(), get_open_port())
27
local_rank = 0
@@ -55,3 +60,77 @@ class UniProcExecutor(ExecutorBase):
60
61
62
UniProcExecutorAsync = UniProcExecutor
63
+
64
+
65
+class ExecutorWithExternalLauncher(UniProcExecutor):
66
+ """An executor that uses external launchers to launch engines,
67
+ specially designed for torchrun-compatible launchers, for
68
+ offline inference with tensor parallelism.
69
+
70
+ see https://github.com/vllm-project/vllm/issues/11400 for
71
+ the motivation, and examples/offline_inference/torchrun_example.py
72
+ for the usage example.
73
+
74
+ The key idea: although it is tensor-parallel inference, we only
75
+ create one worker per executor, users will launch multiple
76
+ engines with torchrun-compatible launchers, and all these engines
77
+ work together to process the same prompts. When scheduling is
78
+ deterministic, all the engines will generate the same outputs,
79
+ and they don't need to synchronize the states with each other.
80
+ """
81
+ uses_ray: bool = False
82
+
83
+ def _init_executor(self) -> None:
84
+ """Initialize the worker and load the model.
85
+ """
86
+ assert self.vllm_config.parallel_config.pipeline_parallel_size == 1, \
87
+ ("ExecutorWithExternalLauncher does not "
88
+ "support pipeline parallelism.")
89
+ assert self.vllm_config.scheduler_config.delay_factor == 0.0, \
90
+ ("ExecutorWithExternalLauncher needs deterministic "
91
+ "execution, so it"
92
+ "does not support delay_factor in scheduling")
93
+ assert not envs.VLLM_USE_V1, \
94
+ ("V1 architecture cannot guarantee deterministic execution, "
95
+ "so it is not supported in ExecutorWithExternalLauncher.")
96
+ self.driver_worker = WorkerWrapperBase(vllm_config=self.vllm_config,
97
+ rpc_rank=0)
98
+ # engines are launched in torchrun-compatible launchers
99
+ # so we can use the env:// method.
100
+ # required env vars:
101
+ # - RANK
102
+ # - MASTER_ADDR
103
+ # - MASTER_PORT
104
+ distributed_init_method = "env://"
105
+ rank = int(os.environ["RANK"])
106
+ local_rank = rank
107
+ is_driver_worker = True
108
+ kwargs = dict(
109
+ vllm_config=self.vllm_config,
110
+ local_rank=local_rank,
111
+ rank=rank,
112
+ distributed_init_method=distributed_init_method,
113
+ is_driver_worker=is_driver_worker,
114
+ )
115
+ self.collective_rpc("init_worker", args=([kwargs], ))
116
+ self.collective_rpc("init_device")
117
+ self.collective_rpc("load_model")
118
+
119
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
120
+ """
121
+ Determine the number of available KV blocks.
122
+ Add an additional all_reduce to get the min across all ranks.
123
+ Note that even if we have the same `gpu_memory_utilization` and
124
+ `swap_space`, the available memory in every rank might still
125
+ differ because NCCL can take different amounts of memory in
126
+ different ranks. Therefore, it is necessary to test if all ranks
127
+ agree on the same KV cache configuration.
128
+ """
129
+ a, b = super().determine_num_available_blocks()
130
+ from vllm.distributed.parallel_state import get_world_group
131
+ cpu_group = get_world_group().cpu_group
132
+ a_tensor = torch.tensor([a], device="cpu", dtype=torch.int64)
133
+ b_tensor = torch.tensor([b], device="cpu", dtype=torch.int64)
134
+ dist.all_reduce(a_tensor, group=cpu_group, op=dist.ReduceOp.MIN)
135
+ dist.all_reduce(b_tensor, group=cpu_group, op=dist.ReduceOp.MIN)
136
+ return a_tensor.item(), b_tensor.item()
vllm/lora/layers.py
+2
-2
@@ -940,8 +940,8 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
940
return self.base_layer.soft_cap
941
942
@property
943
- def use_gather(self):
944
- return self.base_layer.use_gather
943
+ def use_all_gather(self):
944
+ return self.base_layer.use_all_gather
945
946
@property
947
def org_vocab_size(self):
vllm/model_executor/layers/logits_processor.py
+10
-6
@@ -6,6 +6,7 @@ import torch
6
import torch.nn as nn
7
8
import vllm.envs as envs
9
+from vllm.config import get_current_vllm_config
10
from vllm.distributed import (tensor_model_parallel_all_gather,
11
tensor_model_parallel_gather)
12
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -44,8 +45,10 @@ class LogitsProcessor(nn.Module):
45
self.soft_cap = soft_cap
46
# Whether to use gather or all-gather to gather the logits.
47
47
- self.use_gather = not current_platform.is_tpu(
48
- ) and not envs.VLLM_USE_V1
48
+ parallel_config = get_current_vllm_config().parallel_config
49
+ self.use_all_gather = current_platform.is_tpu() \
50
+ or envs.VLLM_USE_V1 \
51
+ or parallel_config.distributed_executor_backend == "external_launcher" # noqa
52
53
def forward(
54
self,
@@ -88,16 +91,17 @@ class LogitsProcessor(nn.Module):
91
logits = lm_head.linear_method.apply(lm_head,
92
hidden_states,
93
bias=embedding_bias)
91
- if self.use_gather:
92
- # None may be returned for rank > 0
93
- logits = tensor_model_parallel_gather(logits)
94
- else:
94
+
95
+ if self.use_all_gather:
96
# Gather is not supported for some devices such as TPUs.
97
# Use all-gather instead.
98
# NOTE(woosuk): Here, the outputs of every device should not be None
99
# because XLA requires strict SPMD among all devices. Every device
100
# should execute the same operations after gathering the logits.
101
logits = tensor_model_parallel_all_gather(logits)
102
+ else:
103
+ # None may be returned for rank > 0
104
+ logits = tensor_model_parallel_gather(logits)
105
# Remove paddings in vocab (if any).
106
if logits is not None:
107
logits = logits[..., :self.org_vocab_size]
vllm/v1/executor/multiproc_executor.py
+1
-1
@@ -246,7 +246,7 @@ class WorkerProc:
246
ready_path: str,
247
):
248
self.rank = rank
249
- wrapper = WorkerWrapperBase(vllm_config=vllm_config, rank=rank)
249
+ wrapper = WorkerWrapperBase(vllm_config=vllm_config, rpc_rank=rank)
250
# TODO: move `init_worker` to executor level as a collective rpc call
251
all_kwargs: List[Dict] = [
252
{} for _ in range(vllm_config.parallel_config.world_size)
vllm/worker/worker.py
-3
@@ -55,9 +55,6 @@ class Worker(LocalOrDistributedWorkerBase):
55
self.rank = rank
56
self.distributed_init_method = distributed_init_method
57
self.is_driver_worker = is_driver_worker
58
- if is_driver_worker:
59
- assert rank % self.parallel_config.tensor_parallel_size == 0, \
60
- "Driver worker should be rank 0 of tensor parallel group."
58
if self.model_config.trust_remote_code:
59
# note: lazy import to avoid importing torch before initializing
60
from vllm.utils import init_cached_hf_modules
vllm/worker/worker_base.py
+20
-9
@@ -461,7 +461,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
461
462
class WorkerWrapperBase:
463
"""
464
- The whole point of this class is to lazily initialize the worker.
464
+ This class represents one process in an executor/engine. It is responsible
465
+ for lazily initializing the worker and handling the worker's lifecycle.
466
We first instantiate the WorkerWrapper, which remembers the worker module
467
and class name. Then, when we call `update_environment_variables`, and the
468
real initialization happens in `init_worker`.
@@ -470,9 +471,19 @@ class WorkerWrapperBase:
471
def __init__(
472
self,
473
vllm_config: VllmConfig,
473
- rank: int = 0,
474
+ rpc_rank: int = 0,
475
) -> None:
475
- self.rank = rank
476
+ """
477
+ Initialize the worker wrapper with the given vllm_config and rpc_rank.
478
+ Note: rpc_rank is the rank of the worker in the executor. In most cases,
479
+ it is also the rank of the worker in the distributed group. However,
480
+ when multiple executors work together, they can be different.
481
+ e.g. in the case of SPMD-style offline inference with TP=2,
482
+ users can launch 2 engines/executors, each with only 1 worker.
483
+ All workers have rpc_rank=0, but they have different ranks in the TP
484
+ group.
485
+ """
486
+ self.rpc_rank = rpc_rank
487
self.vllm_config = vllm_config
488
self.worker: Optional[WorkerBase] = None
489
if vllm_config.model_config is not None:
@@ -485,16 +496,16 @@ class WorkerWrapperBase:
496
497
def adjust_rank(self, rank_mapping: Dict[int, int]) -> None:
498
"""
488
- Adjust the rank based on the given mapping.
499
+ Adjust the rpc_rank based on the given mapping.
500
It is only used during the initialization of the executor,
490
- to adjust the rank of workers after we create all workers.
501
+ to adjust the rpc_rank of workers after we create all workers.
502
"""
492
- if self.rank in rank_mapping:
493
- self.rank = rank_mapping[self.rank]
503
+ if self.rpc_rank in rank_mapping:
504
+ self.rpc_rank = rank_mapping[self.rpc_rank]
505
506
def update_environment_variables(self, envs_list: List[Dict[str,
507
str]]) -> None:
497
- envs = envs_list[self.rank]
508
+ envs = envs_list[self.rpc_rank]
509
key = 'CUDA_VISIBLE_DEVICES'
510
if key in envs and key in os.environ:
511
# overwriting CUDA_VISIBLE_DEVICES is desired behavior
@@ -507,7 +518,7 @@ class WorkerWrapperBase:
518
Here we inject some common logic before initializing the worker.
519
Arguments are passed to the worker class constructor.
520
"""
510
- kwargs = all_kwargs[self.rank]
521
+ kwargs = all_kwargs[self.rpc_rank]
522
enable_trace_function_call_for_thread(self.vllm_config)
523
524
# see https://github.com/NVIDIA/nccl/issues/1234