@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2d18256e47

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Move `ParallelConfig` from `config/init.py` to `config/parallel.py` (#22565)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Aug 9, 2025 at 16:33 UTC 2d18256e47805bf32d1ae04ba1a8c9fd98261fcf

4 files changed +379 -357

.github/CODEOWNERS

+1 -1

@@ -20,7 +20,7 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson

21 # Any change to the VllmConfig changes can have a large user-facing impact,

22 # so spam a lot of people

23 -/vllm/config.py @simon-mo @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @hmellor

23 +/vllm/config @simon-mo @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @hmellor

25 # vLLM V1

26 /vllm/v1 @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @comaniac @alexm-redhat

vllm/config/__init__.py

+2 -355

@@ -25,13 +25,13 @@ from pydantic import (ConfigDict, SkipValidation, field_validator,

25 model_validator)

26 from pydantic.dataclasses import dataclass

27 from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE

28 -from torch.distributed import ProcessGroup, ReduceOp

28 from typing_extensions import Self, assert_never, runtime_checkable

30 import vllm.envs as envs

31 from vllm import version

32 from vllm.config.compilation import (CompilationConfig, CompilationLevel,

33 PassConfig)

34 +from vllm.config.parallel import DistributedExecutorBackend, ParallelConfig

35 from vllm.config.utils import ConfigType, config

36 from vllm.logger import init_logger

37 from vllm.model_executor.layers.quantization import QuantizationMethods

@@ -50,20 +50,16 @@ from vllm.utils import (DEFAULT_MAX_NUM_BATCHED_TOKENS,

50 MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,

51 POOLING_MODEL_MAX_NUM_BATCHED_TOKENS, GiB_bytes,

52 LayerBlockType, LazyLoader, common_broadcastable_dtype,

53 - cuda_device_count_stateless, get_cpu_memory,

54 - get_open_port, random_uuid)

53 + get_cpu_memory, random_uuid)

55 # yapf: enable

57 if TYPE_CHECKING:

58 from _typeshed import DataclassInstance

60 - from ray.runtime_env import RuntimeEnv

61 - from ray.util.placement_group import PlacementGroup

59 from transformers.configuration_utils import PretrainedConfig

61 import vllm.model_executor.layers.quantization as me_quant

62 import vllm.model_executor.models as me_models

66 - from vllm.executor.executor_base import ExecutorBase

63 from vllm.model_executor.layers.quantization import QuantizationMethods

64 from vllm.model_executor.layers.quantization.base_config import (

65 QuantizationConfig)

@@ -73,10 +69,7 @@ if TYPE_CHECKING:

69 HfOverrides = Union[dict, Callable[[type], type]]

70 else:

71 DataclassInstance = Any

76 - PlacementGroup = Any

77 - RuntimeEnv = Any

72 PretrainedConfig = Any

79 - ExecutorBase = Any

73 QuantizationConfig = Any

74 QuantizationMethods = Any

75 BaseModelLoader = Any

@@ -2043,352 +2036,6 @@ class LoadConfig:

2036 self.ignore_patterns = ["original/**/*"]

2037

2038

2046 -DistributedExecutorBackend = Literal["ray", "mp", "uni", "external_launcher"]

2047 -

2048 -

2049 -@config

2050 -@dataclass

2051 -class ParallelConfig:

2052 - """Configuration for the distributed execution."""

2053 -

2054 - pipeline_parallel_size: int = 1

2055 - """Number of pipeline parallel groups."""

2056 - tensor_parallel_size: int = 1

2057 - """Number of tensor parallel groups."""

2058 - data_parallel_size: int = 1

2059 - """Number of data parallel groups. MoE layers will be sharded according to

2060 - the product of the tensor parallel size and data parallel size."""

2061 - data_parallel_size_local: int = 1

2062 - """Number of local data parallel groups."""

2063 - data_parallel_rank: int = 0

2064 - """Rank of the data parallel group."""

2065 - data_parallel_rank_local: Optional[int] = None

2066 - """Local rank of the data parallel group,

2067 - set only in SPMD mode."""

2068 - data_parallel_master_ip: str = "127.0.0.1"

2069 - """IP of the data parallel master."""

2070 - data_parallel_rpc_port: int = 29550

2071 - """Port for data parallel messaging."""

2072 - data_parallel_master_port: int = 29500

2073 - """Port of the data parallel master."""

2074 - data_parallel_backend: str = "mp"

2075 - """Backend to use for data parallel, either "mp" or "ray"."""

2076 - data_parallel_external_lb: bool = False

2077 - """Whether to use "external" DP LB mode. Applies only to online serving

2078 - and when data_parallel_size > 0. This is useful for a "one-pod-per-rank"

2079 - wide-EP setup in Kuberentes. Set implicitly when --data-parallel-rank

2080 - is provided explicitly to vllm serve."""

2081 - data_parallel_hybrid_lb: bool = False

2082 - """Whether to use "hybrid" DP LB mode. Applies only to online serving

2083 - and when data_parallel_size > 0. Enables running an AsyncLLM

2084 - and API server on a "per-node" basis where vLLM load balances

2085 - between local data parallel ranks, but an external LB balances

2086 - between vLLM nodes/replicas. Set explicitly in conjunction with

2087 - --data-parallel-start-rank."""

2088 - enable_expert_parallel: bool = False

2089 - """Use expert parallelism instead of tensor parallelism for MoE layers."""

2090 - enable_eplb: bool = False

2091 - """Enable expert parallelism load balancing for MoE layers."""

2092 - num_redundant_experts: int = 0

2093 - """Number of redundant experts to use for expert parallelism."""

2094 - eplb_window_size: int = 1000

2095 - """Window size for expert load recording."""

2096 - eplb_step_interval: int = 3000

2097 - """

2098 - Interval for rearranging experts in expert parallelism.

2099 -

2100 - Note that if this is greater than the EPLB window size, only the metrics

2101 - of the last `eplb_window_size` steps will be used for rearranging experts.

2102 - """

2103 - eplb_log_balancedness: bool = False

2104 - """

2105 - Log the balancedness each step of expert parallelism.

2106 - This is turned off by default since it will cause communication overhead.

2107 - """

2108 -

2109 - max_parallel_loading_workers: Optional[int] = None

2110 - """Maximum number of parallel loading workers when loading model

2111 - sequentially in multiple batches. To avoid RAM OOM when using tensor

2112 - parallel and large models."""

2113 -

2114 - disable_custom_all_reduce: bool = False

2115 - """Disable the custom all-reduce kernel and fall back to NCCL."""

2116 -

2117 - ray_workers_use_nsight: bool = False

2118 - """Whether to profile Ray workers with nsight, see https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler."""

2119 -

2120 - ray_runtime_env: Optional["RuntimeEnv"] = None

2121 - """Ray runtime environment to pass to distributed workers."""

2122 -

2123 - placement_group: Optional["PlacementGroup"] = None

2124 - """ray distributed model workers placement group."""

2125 -

2126 - distributed_executor_backend: Optional[Union[DistributedExecutorBackend,

2127 - type["ExecutorBase"]]] = None

2128 - """Backend to use for distributed model

2129 - workers, either "ray" or "mp" (multiprocessing). If the product

2130 - of pipeline_parallel_size and tensor_parallel_size is less than

2131 - or equal to the number of GPUs available, "mp" will be used to

2132 - keep processing on a single host. Otherwise, this will default

2133 - to "ray" if Ray is installed and fail otherwise. Note that tpu

2134 - only support Ray for distributed inference."""

2135 -

2136 - worker_cls: str = "auto"

2137 - """The full name of the worker class to use. If "auto", the worker class

2138 - will be determined based on the platform."""

2139 - sd_worker_cls: str = "auto"

2140 - """The full name of the worker class to use for speculative decoding.

2141 - If "auto", the worker class will be determined based on the platform."""

2142 - worker_extension_cls: str = ""

2143 - """The full name of the worker extension class to use. The worker extension

2144 - class is dynamically inherited by the worker class. This is used to inject

2145 - new attributes and methods to the worker class for use in collective_rpc

2146 - calls."""

2147 -

2148 - world_size: int = field(init=False)

2149 - """world_size is TPxPP, it affects the number of workers we create."""

2150 -

2151 - rank: int = 0

2152 - """Global rank in distributed setup."""

2153 -

2154 - enable_multimodal_encoder_data_parallel: bool = False

2155 - """ Use data parallelism instead of tensor parallelism for vision encoder.

2156 - Only support LLama4 for now"""

2157 -

2158 - @property

2159 - def world_size_across_dp(self) -> int:

2160 - """world_size_across_dp is TPxPPxDP, it is the size of the world

2161 - including data parallelism."""

2162 - return self.world_size * self.data_parallel_size

2163 -

2164 - def get_next_dp_init_port(self) -> int:

2165 - """

2166 - We might need to initialize process groups in multiple

2167 - processes that is related to data parallelism,

2168 - e.g. both in the worker and in the engine, which

2169 - can live in different processes. To avoid port conflicts, we

2170 - increment the port number each time we need to initialize a

2171 - new process group related to data parallelism.

2172 - """

2173 - answer = self.data_parallel_master_port

2174 - self.data_parallel_master_port += 1

2175 - return answer

2176 -

2177 - def stateless_init_dp_group(self) -> "ProcessGroup":

2178 - # NOTE: In high-concurrency scenarios multiple processes

2179 - # can pick the same (currently free) port through a race

2180 - # condition when calling `get_open_port()`. When the first

2181 - # process binds the port the others will subsequently fail

2182 - # with `torch.distributed.DistNetworkError: EADDRINUSE`.

2183 - # To make the initialization more robust we retry a few times

2184 - # with a fresh port whenever this specific error is observed.

2185 - from torch.distributed import DistNetworkError

2186 -

2187 - from vllm.distributed.utils import (

2188 - stateless_init_torch_distributed_process_group)

2189 -

2190 - max_retries = 5

2191 - last_exc: Optional[Exception] = None

2192 - for _ in range(max_retries):

2193 - try:

2194 - # use gloo since the engine process might not have cuda device

2195 - return stateless_init_torch_distributed_process_group(

2196 - self.data_parallel_master_ip,

2197 - self.get_next_dp_init_port(),

2198 - self.data_parallel_rank,

2199 - self.data_parallel_size,

2200 - backend="gloo")

2201 - except DistNetworkError as e:

2202 - # We only want to retry when the root cause is EADDRINUSE.

2203 - if "EADDRINUSE" in str(e):

2204 - logger.warning(

2205 - "Address already in use. Retrying with a new port.")

2206 - last_exc = e

2207 - continue # try again with a new port

2208 - raise e

2209 -

2210 - # If we get here all retries have failed.

2211 - assert last_exc is not None

2212 - raise last_exc

2213 -

2214 - @staticmethod

2215 - def has_unfinished_dp(dp_group: "ProcessGroup",

2216 - has_unfinished: bool) -> bool:

2217 - tensor = torch.tensor([has_unfinished],

2218 - dtype=torch.int32,

2219 - device="cpu")

2220 - # dp rank 0: has_unfinished_seqs=True

2221 - # dp rank 1: has_unfinished_seqs=False

2222 - # aggregated: has_unfinished_seqs=True

2223 - # so this is an OR operation, i.e. MAX in integers

2224 - torch.distributed.all_reduce(tensor, op=ReduceOp.MAX, group=dp_group)

2225 - aggregated_has_unfinished = bool(tensor.item())

2226 - return aggregated_has_unfinished

2227 -

2228 - @staticmethod

2229 - def sync_kv_cache_memory_size(dp_group: "ProcessGroup",

2230 - kv_cache_memory: int) -> int:

2231 - if kv_cache_memory == -1:

2232 - kv_cache_memory = torch.iinfo(torch.int64).max

2233 - tensor = torch.tensor([kv_cache_memory],

2234 - dtype=torch.int64,

2235 - device="cpu")

2236 - # we cannot use broadcast for stateless dp group since it depends

2237 - # on global rank

2238 - torch.distributed.all_reduce(tensor, op=ReduceOp.MIN, group=dp_group)

2239 - return tensor.item()

2240 -

2241 - def compute_hash(self):

2242 - """

2243 - Provide a hash that uniquely identifies all the configs

2244 - that affect the structure of the computation

2245 - graph from input ids/embeddings to the final hidden states,

2246 - excluding anything before input ids/embeddings and after

2247 - the final hidden states.

2248 - """

2249 - factors: list[Any] = []

2250 - factors.append(self.pipeline_parallel_size)

2251 - factors.append(self.tensor_parallel_size)

2252 - factors.append(self.enable_expert_parallel)

2253 - factors.append(self.data_parallel_size)

2254 - factors.append(envs.VLLM_ALL2ALL_BACKEND)

2255 - return hashlib.sha256(str(factors).encode()).hexdigest()

2256 -

2257 - def __post_init__(self) -> None:

2258 - self.world_size = self.pipeline_parallel_size * \

2259 - self.tensor_parallel_size

2260 -

2261 - if self.data_parallel_size_local > self.data_parallel_size:

2262 - raise ValueError(

2263 - f"data_parallel_size_local ({self.data_parallel_size_local}) "

2264 - f"must be <= data_parallel_size ({self.data_parallel_size})")

2265 -

2266 - if self.data_parallel_size > 1 or self.data_parallel_size_local == 0:

2267 - # Data parallel was specified in the engine args.

2268 - self.data_parallel_master_port = get_open_port()

2269 -

2270 - if not (0 <= self.data_parallel_rank < self.data_parallel_size):

2271 - raise ValueError(

2272 - f"data_parallel_rank ({self.data_parallel_rank})"

2273 - f" must be in the range [0, {self.data_parallel_size})")

2274 - else:

2275 - # Otherwise fall back to env vars (e.g. for offline SPMD case).

2276 - self.data_parallel_size = envs.VLLM_DP_SIZE

2277 - self.data_parallel_rank = envs.VLLM_DP_RANK

2278 - self.data_parallel_rank_local = envs.VLLM_DP_RANK_LOCAL

2279 - self.data_parallel_master_ip = envs.VLLM_DP_MASTER_IP

2280 - self.data_parallel_master_port = envs.VLLM_DP_MASTER_PORT

2281 -

2282 - if self.data_parallel_external_lb:

2283 - raise ValueError("data_parallel_external_lb can only "

2284 - "be set when data_parallel_size > 1")

2285 -

2286 - if self.distributed_executor_backend == "external_launcher":

2287 - import os

2288 - os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"

2289 - logger.info("Disabling V1 multiprocessing for external launcher.")

2290 -

2291 - if self.enable_eplb:

2292 - if not current_platform.is_cuda():

2293 - raise ValueError(

2294 - "Expert parallelism load balancing is only supported on "

2295 - "CUDA devices now.")

2296 - if self.num_redundant_experts < 0:

2297 - raise ValueError(

2298 - "num_redundant_experts must be non-negative, but got "

2299 - f"{self.num_redundant_experts}.")

2300 - if not self.enable_expert_parallel:

2301 - raise ValueError(

2302 - "enable_expert_parallel must be True to use EPLB.")

2303 - if self.tensor_parallel_size * self.data_parallel_size <= 1:

2304 - raise ValueError(

2305 - "EPLB requires tensor_parallel_size or data_parallel_size "

2306 - f"to be greater than 1, but got "

2307 - f"TP={self.tensor_parallel_size},DP={self.data_parallel_size}."

2308 - )

2309 - else:

2310 - if self.num_redundant_experts != 0:

2311 - raise ValueError(

2312 - "num_redundant_experts should be used with EPLB."

2313 - f"{self.num_redundant_experts}.")

2314 - if self.distributed_executor_backend is None and self.world_size > 1:

2315 - # We use multiprocessing by default if world_size fits on the

2316 - # current node and we aren't in a ray placement group.

2317 -

2318 - from vllm.executor import ray_utils

2319 - backend: DistributedExecutorBackend = "mp"

2320 - ray_found = ray_utils.ray_is_available()

2321 - if current_platform.is_neuron():

2322 - # neuron uses single process to control multiple devices

2323 - backend = "uni"

2324 - elif current_platform.is_tpu() and envs.VLLM_XLA_USE_SPMD:

2325 - backend = "uni"

2326 - elif (current_platform.is_cuda()

2327 - and cuda_device_count_stateless() < self.world_size):

2328 - if not ray_found:

2329 - raise ValueError("Unable to load Ray: "

2330 - f"{ray_utils.ray_import_err}. Ray is "

2331 - "required for multi-node inference, "

2332 - "please install Ray with `pip install "

2333 - "ray`.")

2334 - backend = "ray"

2335 - elif self.data_parallel_backend == "ray":

2336 - logger.info("Using ray distributed inference because "

2337 - "data_parallel_backend is ray")

2338 - backend = "ray"

2339 - elif ray_found:

2340 - if self.placement_group:

2341 - backend = "ray"

2342 - else:

2343 - from ray import is_initialized as ray_is_initialized

2344 - if ray_is_initialized():

2345 - from ray.util import get_current_placement_group

2346 - if get_current_placement_group():

2347 - backend = "ray"

2348 - self.distributed_executor_backend = backend

2349 - logger.debug("Defaulting to use %s for distributed inference",

2350 - backend)

2351 -

2352 - if self.distributed_executor_backend is None and self.world_size == 1:

2353 - self.distributed_executor_backend = "uni"

2354 -

2355 - @property

2356 - def use_ray(self) -> bool:

2357 - return self.distributed_executor_backend == "ray" or (

2358 - isinstance(self.distributed_executor_backend, type)

2359 - and self.distributed_executor_backend.uses_ray)

2360 -

2361 - @model_validator(mode='after')

2362 - def _verify_args(self) -> Self:

2363 - # Lazy import to avoid circular import

2364 - from vllm.executor.executor_base import ExecutorBase

2365 - from vllm.platforms import current_platform

2366 - if self.distributed_executor_backend not in (

2367 - "ray", "mp", "uni",

2368 - "external_launcher", None) and not (isinstance(

2369 - self.distributed_executor_backend, type) and issubclass(

2370 - self.distributed_executor_backend, ExecutorBase)):

2371 - raise ValueError(

2372 - "Unrecognized distributed executor backend "

2373 - f"{self.distributed_executor_backend}. Supported "

2374 - "values are 'ray', 'mp' 'uni', 'external_launcher' or"

2375 - " custom ExecutorBase subclass.")

2376 - if self.use_ray:

2377 - from vllm.executor import ray_utils

2378 - ray_utils.assert_ray_available()

2379 -

2380 - if not current_platform.use_custom_allreduce():

2381 - self.disable_custom_all_reduce = True

2382 - logger.debug(

2383 - "Disabled the custom all-reduce kernel because it is not "

2384 - "supported on current platform.")

2385 - if self.ray_workers_use_nsight and not self.use_ray:

2386 - raise ValueError("Unable to use nsight profiling unless workers "

2387 - "run with Ray.")

2388 -

2389 - return self

2390 -

2391 -

2039 PreemptionMode = Literal["swap", "recompute"]

2040 SchedulerPolicy = Literal["fcfs", "priority"]

2041

vllm/config/compilation.py

+1 -1

@@ -16,7 +16,7 @@ from vllm.logger import init_logger

16 from vllm.utils import is_torch_equal_or_newer, resolve_obj_by_qualname

18 if TYPE_CHECKING:

19 - from vllm.config.config import VllmConfig

19 + from vllm.config import VllmConfig

20 else:

21 VllmConfig = object

vllm/config/parallel.py new

+375

@@ -0,0 +1,375 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import hashlib

5 +from dataclasses import field

6 +from typing import TYPE_CHECKING, Any, Literal, Optional, Union

7 +

8 +import torch

9 +from pydantic import model_validator

10 +from pydantic.dataclasses import dataclass

11 +from torch.distributed import ProcessGroup, ReduceOp

12 +from typing_extensions import Self

13 +

14 +import vllm.envs as envs

15 +from vllm.config.utils import config

16 +from vllm.logger import init_logger

17 +from vllm.platforms import current_platform

18 +from vllm.utils import cuda_device_count_stateless, get_open_port

19 +

20 +if TYPE_CHECKING:

21 + from ray.runtime_env import RuntimeEnv

22 + from ray.util.placement_group import PlacementGroup

23 +

24 + from vllm.executor.executor_base import ExecutorBase

25 +else:

26 + RuntimeEnv = Any

27 + PlacementGroup = Any

28 + ExecutorBase = Any

29 +

30 +logger = init_logger(__name__)

31 +

32 +DistributedExecutorBackend = Literal["ray", "mp", "uni", "external_launcher"]

33 +

34 +

35 +@config

36 +@dataclass

37 +class ParallelConfig:

38 + """Configuration for the distributed execution."""

39 +

40 + pipeline_parallel_size: int = 1

41 + """Number of pipeline parallel groups."""

42 + tensor_parallel_size: int = 1

43 + """Number of tensor parallel groups."""

44 + data_parallel_size: int = 1

45 + """Number of data parallel groups. MoE layers will be sharded according to

46 + the product of the tensor parallel size and data parallel size."""

47 + data_parallel_size_local: int = 1

48 + """Number of local data parallel groups."""

49 + data_parallel_rank: int = 0

50 + """Rank of the data parallel group."""

51 + data_parallel_rank_local: Optional[int] = None

52 + """Local rank of the data parallel group,

53 + set only in SPMD mode."""

54 + data_parallel_master_ip: str = "127.0.0.1"

55 + """IP of the data parallel master."""

56 + data_parallel_rpc_port: int = 29550

57 + """Port for data parallel messaging."""

58 + data_parallel_master_port: int = 29500

59 + """Port of the data parallel master."""

60 + data_parallel_backend: str = "mp"

61 + """Backend to use for data parallel, either "mp" or "ray"."""

62 + data_parallel_external_lb: bool = False

63 + """Whether to use "external" DP LB mode. Applies only to online serving

64 + and when data_parallel_size > 0. This is useful for a "one-pod-per-rank"

65 + wide-EP setup in Kuberentes. Set implicitly when --data-parallel-rank

66 + is provided explicitly to vllm serve."""

67 + data_parallel_hybrid_lb: bool = False

68 + """Whether to use "hybrid" DP LB mode. Applies only to online serving

69 + and when data_parallel_size > 0. Enables running an AsyncLLM

70 + and API server on a "per-node" basis where vLLM load balances

71 + between local data parallel ranks, but an external LB balances

72 + between vLLM nodes/replicas. Set explicitly in conjunction with

73 + --data-parallel-start-rank."""

74 + enable_expert_parallel: bool = False

75 + """Use expert parallelism instead of tensor parallelism for MoE layers."""

76 + enable_eplb: bool = False

77 + """Enable expert parallelism load balancing for MoE layers."""

78 + num_redundant_experts: int = 0

79 + """Number of redundant experts to use for expert parallelism."""

80 + eplb_window_size: int = 1000

81 + """Window size for expert load recording."""

82 + eplb_step_interval: int = 3000

83 + """

84 + Interval for rearranging experts in expert parallelism.

85 +

86 + Note that if this is greater than the EPLB window size, only the metrics

87 + of the last `eplb_window_size` steps will be used for rearranging experts.

88 + """

89 + eplb_log_balancedness: bool = False

90 + """

91 + Log the balancedness each step of expert parallelism.

92 + This is turned off by default since it will cause communication overhead.

93 + """

94 +

95 + max_parallel_loading_workers: Optional[int] = None

96 + """Maximum number of parallel loading workers when loading model

97 + sequentially in multiple batches. To avoid RAM OOM when using tensor

98 + parallel and large models."""

99 +

100 + disable_custom_all_reduce: bool = False

101 + """Disable the custom all-reduce kernel and fall back to NCCL."""

102 +

103 + ray_workers_use_nsight: bool = False

104 + """Whether to profile Ray workers with nsight, see https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler."""

105 +

106 + ray_runtime_env: Optional[RuntimeEnv] = None

107 + """Ray runtime environment to pass to distributed workers."""

108 +

109 + placement_group: Optional[PlacementGroup] = None

110 + """ray distributed model workers placement group."""

111 +

112 + distributed_executor_backend: Optional[Union[DistributedExecutorBackend,

113 + type[ExecutorBase]]] = None

114 + """Backend to use for distributed model

115 + workers, either "ray" or "mp" (multiprocessing). If the product

116 + of pipeline_parallel_size and tensor_parallel_size is less than

117 + or equal to the number of GPUs available, "mp" will be used to

118 + keep processing on a single host. Otherwise, this will default

119 + to "ray" if Ray is installed and fail otherwise. Note that tpu

120 + only support Ray for distributed inference."""

121 +

122 + worker_cls: str = "auto"

123 + """The full name of the worker class to use. If "auto", the worker class

124 + will be determined based on the platform."""

125 + sd_worker_cls: str = "auto"

126 + """The full name of the worker class to use for speculative decoding.

127 + If "auto", the worker class will be determined based on the platform."""

128 + worker_extension_cls: str = ""

129 + """The full name of the worker extension class to use. The worker extension

130 + class is dynamically inherited by the worker class. This is used to inject

131 + new attributes and methods to the worker class for use in collective_rpc

132 + calls."""

133 +

134 + world_size: int = field(init=False)

135 + """world_size is TPxPP, it affects the number of workers we create."""

136 +

137 + rank: int = 0

138 + """Global rank in distributed setup."""

139 +

140 + enable_multimodal_encoder_data_parallel: bool = False

141 + """ Use data parallelism instead of tensor parallelism for vision encoder.

142 + Only support LLama4 for now"""

143 +

144 + @property

145 + def world_size_across_dp(self) -> int:

146 + """world_size_across_dp is TPxPPxDP, it is the size of the world

147 + including data parallelism."""

148 + return self.world_size * self.data_parallel_size

149 +

150 + def get_next_dp_init_port(self) -> int:

151 + """

152 + We might need to initialize process groups in multiple

153 + processes that is related to data parallelism,

154 + e.g. both in the worker and in the engine, which

155 + can live in different processes. To avoid port conflicts, we

156 + increment the port number each time we need to initialize a

157 + new process group related to data parallelism.

158 + """

159 + answer = self.data_parallel_master_port

160 + self.data_parallel_master_port += 1

161 + return answer

162 +

163 + def stateless_init_dp_group(self) -> ProcessGroup:

164 + # NOTE: In high-concurrency scenarios multiple processes

165 + # can pick the same (currently free) port through a race

166 + # condition when calling `get_open_port()`. When the first

167 + # process binds the port the others will subsequently fail

168 + # with `torch.distributed.DistNetworkError: EADDRINUSE`.

169 + # To make the initialization more robust we retry a few times

170 + # with a fresh port whenever this specific error is observed.

171 + from torch.distributed import DistNetworkError

172 +

173 + from vllm.distributed.utils import (

174 + stateless_init_torch_distributed_process_group)

175 +

176 + max_retries = 5

177 + last_exc: Optional[Exception] = None

178 + for _ in range(max_retries):

179 + try:

180 + # use gloo since the engine process might not have cuda device

181 + return stateless_init_torch_distributed_process_group(

182 + self.data_parallel_master_ip,

183 + self.get_next_dp_init_port(),

184 + self.data_parallel_rank,

185 + self.data_parallel_size,

186 + backend="gloo")

187 + except DistNetworkError as e:

188 + # We only want to retry when the root cause is EADDRINUSE.

189 + if "EADDRINUSE" in str(e):

190 + logger.warning(

191 + "Address already in use. Retrying with a new port.")

192 + last_exc = e

193 + continue # try again with a new port

194 + raise e

195 +

196 + # If we get here all retries have failed.

197 + assert last_exc is not None

198 + raise last_exc

199 +

200 + @staticmethod

201 + def has_unfinished_dp(dp_group: ProcessGroup,

202 + has_unfinished: bool) -> bool:

203 + tensor = torch.tensor([has_unfinished],

204 + dtype=torch.int32,

205 + device="cpu")

206 + # dp rank 0: has_unfinished_seqs=True

207 + # dp rank 1: has_unfinished_seqs=False

208 + # aggregated: has_unfinished_seqs=True

209 + # so this is an OR operation, i.e. MAX in integers

210 + torch.distributed.all_reduce(tensor, op=ReduceOp.MAX, group=dp_group)

211 + aggregated_has_unfinished = bool(tensor.item())

212 + return aggregated_has_unfinished

213 +

214 + @staticmethod

215 + def sync_kv_cache_memory_size(dp_group: ProcessGroup,

216 + kv_cache_memory: int) -> int:

217 + if kv_cache_memory == -1:

218 + kv_cache_memory = torch.iinfo(torch.int64).max

219 + tensor = torch.tensor([kv_cache_memory],

220 + dtype=torch.int64,

221 + device="cpu")

222 + # we cannot use broadcast for stateless dp group since it depends

223 + # on global rank

224 + torch.distributed.all_reduce(tensor, op=ReduceOp.MIN, group=dp_group)

225 + return tensor.item()

226 +

227 + def compute_hash(self):

228 + """

229 + Provide a hash that uniquely identifies all the configs

230 + that affect the structure of the computation

231 + graph from input ids/embeddings to the final hidden states,

232 + excluding anything before input ids/embeddings and after

233 + the final hidden states.

234 + """

235 + factors: list[Any] = []

236 + factors.append(self.pipeline_parallel_size)

237 + factors.append(self.tensor_parallel_size)

238 + factors.append(self.enable_expert_parallel)

239 + factors.append(self.data_parallel_size)

240 + factors.append(envs.VLLM_ALL2ALL_BACKEND)

241 + return hashlib.sha256(str(factors).encode()).hexdigest()

242 +

243 + def __post_init__(self) -> None:

244 + self.world_size = self.pipeline_parallel_size * \

245 + self.tensor_parallel_size

246 +

247 + if self.data_parallel_size_local > self.data_parallel_size:

248 + raise ValueError(

249 + f"data_parallel_size_local ({self.data_parallel_size_local}) "

250 + f"must be <= data_parallel_size ({self.data_parallel_size})")

251 +

252 + if self.data_parallel_size > 1 or self.data_parallel_size_local == 0:

253 + # Data parallel was specified in the engine args.

254 + self.data_parallel_master_port = get_open_port()

255 +

256 + if not (0 <= self.data_parallel_rank < self.data_parallel_size):

257 + raise ValueError(

258 + f"data_parallel_rank ({self.data_parallel_rank})"

259 + f" must be in the range [0, {self.data_parallel_size})")

260 + else:

261 + # Otherwise fall back to env vars (e.g. for offline SPMD case).

262 + self.data_parallel_size = envs.VLLM_DP_SIZE

263 + self.data_parallel_rank = envs.VLLM_DP_RANK

264 + self.data_parallel_rank_local = envs.VLLM_DP_RANK_LOCAL

265 + self.data_parallel_master_ip = envs.VLLM_DP_MASTER_IP

266 + self.data_parallel_master_port = envs.VLLM_DP_MASTER_PORT

267 +

268 + if self.data_parallel_external_lb:

269 + raise ValueError("data_parallel_external_lb can only "

270 + "be set when data_parallel_size > 1")

271 +

272 + if self.distributed_executor_backend == "external_launcher":

273 + import os

274 + os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"

275 + logger.info("Disabling V1 multiprocessing for external launcher.")

276 +

277 + if self.enable_eplb:

278 + if not current_platform.is_cuda():

279 + raise ValueError(

280 + "Expert parallelism load balancing is only supported on "

281 + "CUDA devices now.")

282 + if self.num_redundant_experts < 0:

283 + raise ValueError(

284 + "num_redundant_experts must be non-negative, but got "

285 + f"{self.num_redundant_experts}.")

286 + if not self.enable_expert_parallel:

287 + raise ValueError(

288 + "enable_expert_parallel must be True to use EPLB.")

289 + if self.tensor_parallel_size * self.data_parallel_size <= 1:

290 + raise ValueError(

291 + "EPLB requires tensor_parallel_size or data_parallel_size "

292 + f"to be greater than 1, but got "

293 + f"TP={self.tensor_parallel_size},DP={self.data_parallel_size}."

294 + )

295 + else:

296 + if self.num_redundant_experts != 0:

297 + raise ValueError(

298 + "num_redundant_experts should be used with EPLB."

299 + f"{self.num_redundant_experts}.")

300 + if self.distributed_executor_backend is None and self.world_size > 1:

301 + # We use multiprocessing by default if world_size fits on the

302 + # current node and we aren't in a ray placement group.

303 +

304 + from vllm.executor import ray_utils

305 + backend: DistributedExecutorBackend = "mp"

306 + ray_found = ray_utils.ray_is_available()

307 + if current_platform.is_neuron():

308 + # neuron uses single process to control multiple devices

309 + backend = "uni"

310 + elif current_platform.is_tpu() and envs.VLLM_XLA_USE_SPMD:

311 + backend = "uni"

312 + elif (current_platform.is_cuda()

313 + and cuda_device_count_stateless() < self.world_size):

314 + if not ray_found:

315 + raise ValueError("Unable to load Ray: "

316 + f"{ray_utils.ray_import_err}. Ray is "

317 + "required for multi-node inference, "

318 + "please install Ray with `pip install "

319 + "ray`.")

320 + backend = "ray"

321 + elif self.data_parallel_backend == "ray":

322 + logger.info("Using ray distributed inference because "

323 + "data_parallel_backend is ray")

324 + backend = "ray"

325 + elif ray_found:

326 + if self.placement_group:

327 + backend = "ray"

328 + else:

329 + from ray import is_initialized as ray_is_initialized

330 + if ray_is_initialized():

331 + from ray.util import get_current_placement_group

332 + if get_current_placement_group():

333 + backend = "ray"

334 + self.distributed_executor_backend = backend

335 + logger.debug("Defaulting to use %s for distributed inference",

336 + backend)

337 +

338 + if self.distributed_executor_backend is None and self.world_size == 1:

339 + self.distributed_executor_backend = "uni"

340 +

341 + @property

342 + def use_ray(self) -> bool:

343 + return self.distributed_executor_backend == "ray" or (

344 + isinstance(self.distributed_executor_backend, type)

345 + and self.distributed_executor_backend.uses_ray)

346 +

347 + @model_validator(mode='after')

348 + def _verify_args(self) -> Self:

349 + # Lazy import to avoid circular import

350 + from vllm.executor.executor_base import ExecutorBase

351 + from vllm.platforms import current_platform

352 + if self.distributed_executor_backend not in (

353 + "ray", "mp", "uni",

354 + "external_launcher", None) and not (isinstance(

355 + self.distributed_executor_backend, type) and issubclass(

356 + self.distributed_executor_backend, ExecutorBase)):

357 + raise ValueError(

358 + "Unrecognized distributed executor backend "

359 + f"{self.distributed_executor_backend}. Supported "

360 + "values are 'ray', 'mp' 'uni', 'external_launcher' or"

361 + " custom ExecutorBase subclass.")

362 + if self.use_ray:

363 + from vllm.executor import ray_utils

364 + ray_utils.assert_ray_available()

365 +

366 + if not current_platform.use_custom_allreduce():

367 + self.disable_custom_all_reduce = True

368 + logger.debug(

369 + "Disabled the custom all-reduce kernel because it is not "

370 + "supported on current platform.")

371 + if self.ray_workers_use_nsight and not self.use_ray:

372 + raise ValueError("Unable to use nsight profiling unless workers "

373 + "run with Ray.")

374 +

375 + return self

Move `ParallelConfig` from `config/__init__.py` to `config/parallel.py` (#22565)

Move `ParallelConfig` from `config/init.py` to `config/parallel.py` (#22565)