@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf53e0c70b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

Support torchrun and SPMD-style offline inference (#12071)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 16, 2025 at 19:58 UTC bf53e0c70b0fe17087914cc770fd801e0bf02137
14 files changed +248 -30
.buildkite/test-pipeline.yaml
+1
@@ -463,6 +463,7 @@ steps:
463 - vllm/worker/worker.py
464 - vllm/worker/model_runner.py
465 commands:
466 + - torchrun --nproc-per-node=2 distributed/test_torchrun_example.py
467 - pytest -v -s ./compile/test_basic_correctness.py
468 - pytest -v -s ./compile/test_wrapper.py
469 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
examples/offline_inference/torchrun_example.py new
+64
@@ -0,0 +1,64 @@
1 +"""
2 +experimental support for tensor-parallel inference with torchrun,
3 +see https://github.com/vllm-project/vllm/issues/11400 for
4 +the motivation and use case for this example.
5 +run the script with `torchrun --nproc-per-node=2 torchrun_example.py`,
6 +the argument 2 should match the `tensor_parallel_size` below.
7 +see `tests/distributed/test_torchrun_example.py` for the unit test.
8 +"""
9 +
10 +from vllm import LLM, SamplingParams
11 +
12 +# Create prompts, the same across all ranks
13 +prompts = [
14 + "Hello, my name is",
15 + "The president of the United States is",
16 + "The capital of France is",
17 + "The future of AI is",
18 +]
19 +
20 +# Create sampling parameters, the same across all ranks
21 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
22 +
23 +# Use `distributed_executor_backend="external_launcher"` so that
24 +# this llm engine/instance only creates one worker.
25 +llm = LLM(
26 + model="facebook/opt-125m",
27 + tensor_parallel_size=2,
28 + distributed_executor_backend="external_launcher",
29 +)
30 +
31 +outputs = llm.generate(prompts, sampling_params)
32 +
33 +# all ranks will have the same outputs
34 +for output in outputs:
35 + prompt = output.prompt
36 + generated_text = output.outputs[0].text
37 + print(f"Prompt: {prompt!r}, "
38 + f"Generated text: {generated_text!r}")
39 +"""
40 +Further tips:
41 +
42 +1. to communicate control messages across all ranks, use the cpu group,
43 +a PyTorch ProcessGroup with GLOO backend.
44 +
45 +```python
46 +from vllm.distributed.parallel_state import get_world_group
47 +cpu_group = get_world_group().cpu_group
48 +torch_rank = dist.get_rank(group=cpu_group)
49 +if torch_rank == 0:
50 + # do something for rank 0, e.g. saving the results to disk.
51 +```
52 +
53 +2. to communicate data across all ranks, use the model's device group,
54 +a PyTorch ProcessGroup with NCCL backend.
55 +```python
56 +from vllm.distributed.parallel_state import get_world_group
57 +device_group = get_world_group().device_group
58 +```
59 +
60 +3. to access the model directly in every rank, use the following code:
61 +```python
62 +llm.llm_engine.model_executor.driver_worker.worker.model_runner.model
63 +```
64 +"""
tests/distributed/test_torchrun_example.py new
+56
@@ -0,0 +1,56 @@
1 +# unit test for `examples/offline_inference/torchrun_example.py`
2 +
3 +import random
4 +
5 +import torch.distributed as dist
6 +
7 +from vllm import LLM, SamplingParams
8 +from vllm.distributed.parallel_state import get_world_group
9 +
10 +# Create prompts
11 +prompts = [
12 + "Hello, my name is",
13 + "The president of the United States is",
14 + "The capital of France is",
15 + "The future of AI is",
16 +]
17 +
18 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
19 +
20 +# set different `gpu_memory_utilization` and `swap_space` for different ranks,
21 +# to test if all ranks agree on the same kv cache configuration.
22 +llm = LLM(model="facebook/opt-125m",
23 + tensor_parallel_size=2,
24 + distributed_executor_backend="external_launcher",
25 + gpu_memory_utilization=random.uniform(0.7, 0.9),
26 + swap_space=random.randint(1, 4))
27 +
28 +outputs = llm.generate(prompts, sampling_params)
29 +
30 +cpu_group = get_world_group().cpu_group
31 +
32 +torch_rank = dist.get_rank(group=cpu_group)
33 +
34 +
35 +def test_consistent_across_ranks(obj):
36 + if torch_rank == 0:
37 + dist.broadcast_object_list([obj], src=0, group=cpu_group)
38 + else:
39 + container = [None]
40 + dist.broadcast_object_list(container, src=0, group=cpu_group)
41 + assert container[0] == obj
42 +
43 +
44 +test_consistent_across_ranks(
45 + llm.llm_engine.vllm_config.cache_config.num_cpu_blocks)
46 +test_consistent_across_ranks(
47 + llm.llm_engine.vllm_config.cache_config.num_gpu_blocks)
48 +
49 +# all ranks should have the same outputs
50 +for output in outputs:
51 + prompt = output.prompt
52 + generated_text = output.outputs[0].text
53 + test_consistent_across_ranks(prompt)
54 + test_consistent_across_ranks(generated_text)
55 + print(f"Rank {torch_rank}, Prompt: {prompt!r}, "
56 + f"Generated text: {generated_text!r}")
tests/engine/test_multiproc_workers.py
+1 -1
@@ -22,7 +22,7 @@ class DummyWorkerWrapper(WorkerWrapperBase):
22 # simulate error case
23 raise worker_input
24
25 - return self.rank, input
25 + return self.rpc_rank, input
26
27
28 def _start_workers() -> Tuple[List[ProcessWorkerWrapper], WorkerMonitor]:
vllm/config.py
+4 -3
@@ -1338,14 +1338,15 @@ class ParallelConfig:
1338 from vllm.executor.executor_base import ExecutorBase
1339 from vllm.platforms import current_platform
1340 if self.distributed_executor_backend not in (
1341 - "ray", "mp", "uni", None) and not (isinstance(
1341 + "ray", "mp", "uni",
1342 + "external_launcher", None) and not (isinstance(
1343 self.distributed_executor_backend, type) and issubclass(
1344 self.distributed_executor_backend, ExecutorBase)):
1345 raise ValueError(
1346 "Unrecognized distributed executor backend "
1347 f"{self.distributed_executor_backend}. Supported "
1347 - "values are 'ray', 'mp' 'uni', or custom ExecutorBase"
1348 - " subclass.")
1348 + "values are 'ray', 'mp' 'uni', 'external_launcher' or"
1349 + " custom ExecutorBase subclass.")
1350 if self.use_ray:
1351 from vllm.executor import ray_utils
1352 ray_utils.assert_ray_available()
vllm/engine/arg_utils.py
+1 -1
@@ -388,7 +388,7 @@ class EngineArgs:
388 # Parallel arguments
389 parser.add_argument(
390 '--distributed-executor-backend',
391 - choices=['ray', 'mp'],
391 + choices=['ray', 'mp', 'uni', 'external_launcher'],
392 default=EngineArgs.distributed_executor_backend,
393 help='Backend to use for distributed model '
394 'workers, either "ray" or "mp" (multiprocessing). If the product '
vllm/engine/llm_engine.py
+5
@@ -457,6 +457,11 @@ class LLMEngine:
457 # JAX-style, single-process, multi-device executor.
458 from vllm.executor.uniproc_executor import UniProcExecutor
459 executor_class = UniProcExecutor
460 + elif distributed_executor_backend == "external_launcher":
461 + # executor with external launcher
462 + from vllm.executor.uniproc_executor import ( # noqa
463 + ExecutorWithExternalLauncher)
464 + executor_class = ExecutorWithExternalLauncher
465 else:
466 from vllm.executor.uniproc_executor import UniProcExecutor
467 executor_class = UniProcExecutor
vllm/executor/ray_distributed_executor.py
+3 -3
@@ -172,7 +172,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
172 scheduling_strategy=scheduling_strategy,
173 **ray_remote_kwargs,
174 )(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
175 - rank=rank)
175 + rpc_rank=rank)
176 else:
177 worker = ray.remote(
178 num_cpus=0,
@@ -181,7 +181,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
181 scheduling_strategy=scheduling_strategy,
182 **ray_remote_kwargs,
183 )(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
184 - rank=rank)
184 + rpc_rank=rank)
185 worker_metadata.append(
186 RayWorkerMetaData(worker=worker, created_rank=rank))
187 rank += 1
@@ -204,7 +204,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
204 # as the resource holder for the driver process.
205 self.driver_dummy_worker = worker
206 self.driver_worker = RayWorkerWrapper(
207 - vllm_config=self.vllm_config, rank=0)
207 + vllm_config=self.vllm_config, rpc_rank=0)
208 worker_metadata.pop(i)
209 break
210
vllm/executor/uniproc_executor.py
+80 -1
@@ -1,5 +1,10 @@
1 +import os
2 from typing import Any, Dict, List, Optional, Tuple
3
4 +import torch
5 +import torch.distributed as dist
6 +
7 +import vllm.envs as envs
8 from vllm.executor.executor_base import ExecutorBase
9 from vllm.logger import init_logger
10 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
@@ -16,7 +21,7 @@ class UniProcExecutor(ExecutorBase):
21 """Initialize the worker and load the model.
22 """
23 self.driver_worker = WorkerWrapperBase(vllm_config=self.vllm_config,
19 - rank=0)
24 + rpc_rank=0)
25 distributed_init_method = get_distributed_init_method(
26 get_ip(), get_open_port())
27 local_rank = 0
@@ -55,3 +60,77 @@ class UniProcExecutor(ExecutorBase):
60
61
62 UniProcExecutorAsync = UniProcExecutor
63 +
64 +
65 +class ExecutorWithExternalLauncher(UniProcExecutor):
66 + """An executor that uses external launchers to launch engines,
67 + specially designed for torchrun-compatible launchers, for
68 + offline inference with tensor parallelism.
69 +
70 + see https://github.com/vllm-project/vllm/issues/11400 for
71 + the motivation, and examples/offline_inference/torchrun_example.py
72 + for the usage example.
73 +
74 + The key idea: although it is tensor-parallel inference, we only
75 + create one worker per executor, users will launch multiple
76 + engines with torchrun-compatible launchers, and all these engines
77 + work together to process the same prompts. When scheduling is
78 + deterministic, all the engines will generate the same outputs,
79 + and they don't need to synchronize the states with each other.
80 + """
81 + uses_ray: bool = False
82 +
83 + def _init_executor(self) -> None:
84 + """Initialize the worker and load the model.
85 + """
86 + assert self.vllm_config.parallel_config.pipeline_parallel_size == 1, \
87 + ("ExecutorWithExternalLauncher does not "
88 + "support pipeline parallelism.")
89 + assert self.vllm_config.scheduler_config.delay_factor == 0.0, \
90 + ("ExecutorWithExternalLauncher needs deterministic "
91 + "execution, so it"
92 + "does not support delay_factor in scheduling")
93 + assert not envs.VLLM_USE_V1, \
94 + ("V1 architecture cannot guarantee deterministic execution, "
95 + "so it is not supported in ExecutorWithExternalLauncher.")
96 + self.driver_worker = WorkerWrapperBase(vllm_config=self.vllm_config,
97 + rpc_rank=0)
98 + # engines are launched in torchrun-compatible launchers
99 + # so we can use the env:// method.
100 + # required env vars:
101 + # - RANK
102 + # - MASTER_ADDR
103 + # - MASTER_PORT
104 + distributed_init_method = "env://"
105 + rank = int(os.environ["RANK"])
106 + local_rank = rank
107 + is_driver_worker = True
108 + kwargs = dict(
109 + vllm_config=self.vllm_config,
110 + local_rank=local_rank,
111 + rank=rank,
112 + distributed_init_method=distributed_init_method,
113 + is_driver_worker=is_driver_worker,
114 + )
115 + self.collective_rpc("init_worker", args=([kwargs], ))
116 + self.collective_rpc("init_device")
117 + self.collective_rpc("load_model")
118 +
119 + def determine_num_available_blocks(self) -> Tuple[int, int]:
120 + """
121 + Determine the number of available KV blocks.
122 + Add an additional all_reduce to get the min across all ranks.
123 + Note that even if we have the same `gpu_memory_utilization` and
124 + `swap_space`, the available memory in every rank might still
125 + differ because NCCL can take different amounts of memory in
126 + different ranks. Therefore, it is necessary to test if all ranks
127 + agree on the same KV cache configuration.
128 + """
129 + a, b = super().determine_num_available_blocks()
130 + from vllm.distributed.parallel_state import get_world_group
131 + cpu_group = get_world_group().cpu_group
132 + a_tensor = torch.tensor([a], device="cpu", dtype=torch.int64)
133 + b_tensor = torch.tensor([b], device="cpu", dtype=torch.int64)
134 + dist.all_reduce(a_tensor, group=cpu_group, op=dist.ReduceOp.MIN)
135 + dist.all_reduce(b_tensor, group=cpu_group, op=dist.ReduceOp.MIN)
136 + return a_tensor.item(), b_tensor.item()
vllm/lora/layers.py
+2 -2
@@ -940,8 +940,8 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
940 return self.base_layer.soft_cap
941
942 @property
943 - def use_gather(self):
944 - return self.base_layer.use_gather
943 + def use_all_gather(self):
944 + return self.base_layer.use_all_gather
945
946 @property
947 def org_vocab_size(self):
vllm/model_executor/layers/logits_processor.py
+10 -6
@@ -6,6 +6,7 @@ import torch
6 import torch.nn as nn
7
8 import vllm.envs as envs
9 +from vllm.config import get_current_vllm_config
10 from vllm.distributed import (tensor_model_parallel_all_gather,
11 tensor_model_parallel_gather)
12 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -44,8 +45,10 @@ class LogitsProcessor(nn.Module):
45 self.soft_cap = soft_cap
46 # Whether to use gather or all-gather to gather the logits.
47
47 - self.use_gather = not current_platform.is_tpu(
48 - ) and not envs.VLLM_USE_V1
48 + parallel_config = get_current_vllm_config().parallel_config
49 + self.use_all_gather = current_platform.is_tpu() \
50 + or envs.VLLM_USE_V1 \
51 + or parallel_config.distributed_executor_backend == "external_launcher" # noqa
52
53 def forward(
54 self,
@@ -88,16 +91,17 @@ class LogitsProcessor(nn.Module):
91 logits = lm_head.linear_method.apply(lm_head,
92 hidden_states,
93 bias=embedding_bias)
91 - if self.use_gather:
92 - # None may be returned for rank > 0
93 - logits = tensor_model_parallel_gather(logits)
94 - else:
94 +
95 + if self.use_all_gather:
96 # Gather is not supported for some devices such as TPUs.
97 # Use all-gather instead.
98 # NOTE(woosuk): Here, the outputs of every device should not be None
99 # because XLA requires strict SPMD among all devices. Every device
100 # should execute the same operations after gathering the logits.
101 logits = tensor_model_parallel_all_gather(logits)
102 + else:
103 + # None may be returned for rank > 0
104 + logits = tensor_model_parallel_gather(logits)
105 # Remove paddings in vocab (if any).
106 if logits is not None:
107 logits = logits[..., :self.org_vocab_size]
vllm/v1/executor/multiproc_executor.py
+1 -1
@@ -246,7 +246,7 @@ class WorkerProc:
246 ready_path: str,
247 ):
248 self.rank = rank
249 - wrapper = WorkerWrapperBase(vllm_config=vllm_config, rank=rank)
249 + wrapper = WorkerWrapperBase(vllm_config=vllm_config, rpc_rank=rank)
250 # TODO: move `init_worker` to executor level as a collective rpc call
251 all_kwargs: List[Dict] = [
252 {} for _ in range(vllm_config.parallel_config.world_size)
vllm/worker/worker.py
-3
@@ -55,9 +55,6 @@ class Worker(LocalOrDistributedWorkerBase):
55 self.rank = rank
56 self.distributed_init_method = distributed_init_method
57 self.is_driver_worker = is_driver_worker
58 - if is_driver_worker:
59 - assert rank % self.parallel_config.tensor_parallel_size == 0, \
60 - "Driver worker should be rank 0 of tensor parallel group."
58 if self.model_config.trust_remote_code:
59 # note: lazy import to avoid importing torch before initializing
60 from vllm.utils import init_cached_hf_modules
vllm/worker/worker_base.py
+20 -9
@@ -461,7 +461,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
461
462 class WorkerWrapperBase:
463 """
464 - The whole point of this class is to lazily initialize the worker.
464 + This class represents one process in an executor/engine. It is responsible
465 + for lazily initializing the worker and handling the worker's lifecycle.
466 We first instantiate the WorkerWrapper, which remembers the worker module
467 and class name. Then, when we call `update_environment_variables`, and the
468 real initialization happens in `init_worker`.
@@ -470,9 +471,19 @@ class WorkerWrapperBase:
471 def __init__(
472 self,
473 vllm_config: VllmConfig,
473 - rank: int = 0,
474 + rpc_rank: int = 0,
475 ) -> None:
475 - self.rank = rank
476 + """
477 + Initialize the worker wrapper with the given vllm_config and rpc_rank.
478 + Note: rpc_rank is the rank of the worker in the executor. In most cases,
479 + it is also the rank of the worker in the distributed group. However,
480 + when multiple executors work together, they can be different.
481 + e.g. in the case of SPMD-style offline inference with TP=2,
482 + users can launch 2 engines/executors, each with only 1 worker.
483 + All workers have rpc_rank=0, but they have different ranks in the TP
484 + group.
485 + """
486 + self.rpc_rank = rpc_rank
487 self.vllm_config = vllm_config
488 self.worker: Optional[WorkerBase] = None
489 if vllm_config.model_config is not None:
@@ -485,16 +496,16 @@ class WorkerWrapperBase:
496
497 def adjust_rank(self, rank_mapping: Dict[int, int]) -> None:
498 """
488 - Adjust the rank based on the given mapping.
499 + Adjust the rpc_rank based on the given mapping.
500 It is only used during the initialization of the executor,
490 - to adjust the rank of workers after we create all workers.
501 + to adjust the rpc_rank of workers after we create all workers.
502 """
492 - if self.rank in rank_mapping:
493 - self.rank = rank_mapping[self.rank]
503 + if self.rpc_rank in rank_mapping:
504 + self.rpc_rank = rank_mapping[self.rpc_rank]
505
506 def update_environment_variables(self, envs_list: List[Dict[str,
507 str]]) -> None:
497 - envs = envs_list[self.rank]
508 + envs = envs_list[self.rpc_rank]
509 key = 'CUDA_VISIBLE_DEVICES'
510 if key in envs and key in os.environ:
511 # overwriting CUDA_VISIBLE_DEVICES is desired behavior
@@ -507,7 +518,7 @@ class WorkerWrapperBase:
518 Here we inject some common logic before initializing the worker.
519 Arguments are passed to the worker class constructor.
520 """
510 - kwargs = all_kwargs[self.rank]
521 + kwargs = all_kwargs[self.rpc_rank]
522 enable_trace_function_call_for_thread(self.vllm_config)
523
524 # see https://github.com/NVIDIA/nccl/issues/1234