| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """experimental support for data-parallel inference with torchrun |
| 4 | Note the data load balancing and distribution is done out of the vllm engine, |
| 5 | no internal lb supported in external_launcher mode. |
| 6 | |
| 7 | To run this example: |
| 8 | ```bash |
| 9 | $ torchrun --nproc-per-node=2 examples/features/torchrun/torchrun_dp_example_offline.py |
| 10 | ``` |
| 11 | |
| 12 | With custom parallelism settings: |
| 13 | ```bash |
| 14 | $ torchrun --nproc-per-node=8 examples/features/torchrun/torchrun_dp_example_offline.py \ |
| 15 | --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep |
| 16 | ``` |
| 17 | """ # noqa: E501 |
| 18 | |
| 19 | import argparse |
| 20 | |
| 21 | from vllm import LLM, SamplingParams |
| 22 | |
| 23 | |
| 24 | def parse_args(): |
| 25 | parser = argparse.ArgumentParser( |
| 26 | description="Data-parallel inference with torchrun" |
| 27 | ) |
| 28 | parser.add_argument( |
| 29 | "--tp-size", |
| 30 | type=int, |
| 31 | default=1, |
| 32 | help="Tensor parallel size (default: 1)", |
| 33 | ) |
| 34 | parser.add_argument( |
| 35 | "--pp-size", |
| 36 | type=int, |
| 37 | default=1, |
| 38 | help="Pipeline parallel size (default: 1)", |
| 39 | ) |
| 40 | parser.add_argument( |
| 41 | "--dp-size", |
| 42 | type=int, |
| 43 | default=2, |
| 44 | help="Data parallel size (default: 2)", |
| 45 | ) |
| 46 | parser.add_argument( |
| 47 | "--enable-ep", |
| 48 | action="store_true", |
| 49 | help="Enable expert parallel (default: False)", |
| 50 | ) |
| 51 | parser.add_argument( |
| 52 | "--model", |
| 53 | type=str, |
| 54 | default="microsoft/Phi-mini-MoE-instruct", |
| 55 | help="Model name or path (default: microsoft/Phi-mini-MoE-instruct)", |
| 56 | ) |
| 57 | parser.add_argument( |
| 58 | "--max-model-len", |
| 59 | type=int, |
| 60 | default=4096, |
| 61 | help="Maximum model length (default: 4096)", |
| 62 | ) |
| 63 | parser.add_argument( |
| 64 | "--gpu-memory-utilization", |
| 65 | type=float, |
| 66 | default=0.6, |
| 67 | help="GPU memory utilization (default: 0.6)", |
| 68 | ) |
| 69 | parser.add_argument( |
| 70 | "--seed", |
| 71 | type=int, |
| 72 | default=1, |
| 73 | help="Random seed (default: 1)", |
| 74 | ) |
| 75 | return parser.parse_args() |
| 76 | |
| 77 | |
| 78 | args = parse_args() |
| 79 | |
| 80 | |
| 81 | # Create prompts, the same across all ranks |
| 82 | prompts = [ |
| 83 | "Hello, my name is", |
| 84 | "The president of the United States is", |
| 85 | "The capital of France is", |
| 86 | "The future of AI is", |
| 87 | ] |
| 88 | |
| 89 | # Create sampling parameters, the same across all ranks |
| 90 | sampling_params = SamplingParams(temperature=0.8, top_p=0.95) |
| 91 | |
| 92 | # Use `distributed_executor_backend="external_launcher"` so that |
| 93 | # this llm engine/instance only creates one worker. |
| 94 | # it is important to set an explicit seed to make sure that |
| 95 | # all ranks have the same random seed, so that sampling can be |
| 96 | # deterministic across ranks. |
| 97 | llm = LLM( |
| 98 | model=args.model, |
| 99 | tensor_parallel_size=args.tp_size, |
| 100 | data_parallel_size=args.dp_size, |
| 101 | pipeline_parallel_size=args.pp_size, |
| 102 | enable_expert_parallel=args.enable_ep, |
| 103 | distributed_executor_backend="external_launcher", |
| 104 | max_model_len=args.max_model_len, |
| 105 | gpu_memory_utilization=args.gpu_memory_utilization, |
| 106 | seed=args.seed, |
| 107 | ) |
| 108 | |
| 109 | dp_rank = llm.llm_engine.vllm_config.parallel_config.data_parallel_rank |
| 110 | dp_size = llm.llm_engine.vllm_config.parallel_config.data_parallel_size |
| 111 | |
| 112 | prompts = [ |
| 113 | f"{idx}.{prompt}" for idx, prompt in enumerate(prompts) if idx % dp_size == dp_rank |
| 114 | ] |
| 115 | |
| 116 | outputs = llm.generate(prompts, sampling_params) |
| 117 | |
| 118 | for output in outputs: |
| 119 | prompt = output.prompt |
| 120 | generated_text = output.outputs[0].text |
| 121 | print( |
| 122 | f"DP Rank: {dp_rank} Prompt: {prompt!r}\nGenerated text: {generated_text!r}\n" |
| 123 | ) |
| 124 | |
| 125 | """ |
| 126 | Further tips: |
| 127 | |
| 128 | 1. to communicate control messages across all ranks, use the cpu group, |
| 129 | a PyTorch ProcessGroup with GLOO backend. |
| 130 | |
| 131 | ```python |
| 132 | from vllm.distributed.parallel_state import get_world_group |
| 133 | cpu_group = get_world_group().cpu_group |
| 134 | torch_rank = dist.get_rank(group=cpu_group) |
| 135 | if torch_rank == 0: |
| 136 | # do something for rank 0, e.g. saving the results to disk. |
| 137 | ``` |
| 138 | |
| 139 | 2. to communicate data across all ranks, use the model's device group, |
| 140 | a PyTorch ProcessGroup with NCCL backend. |
| 141 | ```python |
| 142 | from vllm.distributed.parallel_state import get_world_group |
| 143 | device_group = get_world_group().device_group |
| 144 | ``` |
| 145 | |
| 146 | 3. to access the model directly in every rank, use the following code: |
| 147 | ```python |
| 148 | llm.llm_engine.model_executor.driver_worker.worker.model_runner.model |
| 149 | ``` |
| 150 | """ |