| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | from dataclasses import dataclass, replace |
| 6 | |
| 7 | import torch |
| 8 | import torch.distributed as dist |
| 9 | |
| 10 | from vllm.config import ParallelConfig |
| 11 | from vllm.config.compilation import CUDAGraphMode |
| 12 | from vllm.distributed.parallel_state import get_dp_group |
| 13 | from vllm.v1.worker.dp_utils import should_skip_dp_coordination |
| 14 | from vllm.v1.worker.gpu.cudagraph_utils import ( |
| 15 | BatchExecutionDescriptor, |
| 16 | CudaGraphManager, |
| 17 | ) |
| 18 | from vllm.v1.worker.ubatch_utils import check_ubatch_thresholds, get_num_ubatches |
| 19 | |
| 20 | |
| 21 | @dataclass(frozen=True) |
| 22 | class DPSyncState: |
| 23 | """What a `dispatch_cg_and_sync_dp` call agreed across DP ranks. |
| 24 | |
| 25 | Every field is identical on every rank, so callers can branch on them |
| 26 | without disagreeing about whether to run a collective. Never add a per-rank |
| 27 | value here. |
| 28 | """ |
| 29 | |
| 30 | # Per-rank token counts, for the forward context. All entries hold the |
| 31 | # agreed padded count, unless `eager`, where nothing was padded. |
| 32 | num_tokens_across_dp: torch.Tensor |
| 33 | # Agreed uniform decode length. None means the ranks disagreed, which is an |
| 34 | # answer, not "unknown". |
| 35 | uniform_token_count: int | None |
| 36 | # Whether the ranks agreed to run eager. A dispatch reusing this must run |
| 37 | # eager too; no shape was agreed, so picking a graph per rank would diverge. |
| 38 | eager: bool |
| 39 | # Agreed upper bound on any rank's request count. Holds the padded count when |
| 40 | # a FULL descriptor imposed one, else the most any rank scheduled. |
| 41 | num_reqs: int |
| 42 | |
| 43 | |
| 44 | def sync_cudagraph_and_dp_padding( |
| 45 | cudagraph_manager: CudaGraphManager | None, |
| 46 | desired_batch_desc: BatchExecutionDescriptor, |
| 47 | num_tokens: int, |
| 48 | num_reqs: int, |
| 49 | uniform_token_count: int | None, |
| 50 | dp_size: int, |
| 51 | dp_rank: int, |
| 52 | max_query_len: int | None = None, |
| 53 | num_active_loras: int = 0, |
| 54 | parallel_config: ParallelConfig | None = None, |
| 55 | allow_ubatching: bool = False, |
| 56 | uniform_decode: bool = False, |
| 57 | ) -> tuple[BatchExecutionDescriptor, DPSyncState | None]: |
| 58 | """Coordinates the batch descriptor and DP padding across all ranks. |
| 59 | |
| 60 | `parallel_config` is only needed to decide whether to microbatch, so callers |
| 61 | that never do (`allow_ubatching=False`) can leave it out. |
| 62 | |
| 63 | Returns (synced_batch_desc, sync). `sync` is None when no rank has work. |
| 64 | """ |
| 65 | assert dp_size > 1, "DP size must be greater than 1" |
| 66 | group = get_dp_group().cpu_group |
| 67 | tensor = torch.zeros(6, dp_size, dtype=torch.int32, device="cpu") |
| 68 | tensor[0][dp_rank] = num_tokens |
| 69 | tensor[1][dp_rank] = desired_batch_desc.cg_mode.value |
| 70 | tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None) |
| 71 | tensor[3][dp_rank] = max_query_len or -1 # (-1 means None) |
| 72 | tensor[4][dp_rank] = int(allow_ubatching) |
| 73 | tensor[5][dp_rank] = num_reqs |
| 74 | if should_skip_dp_coordination(): |
| 75 | tensor[:] = tensor[:, dp_rank, None].clone() |
| 76 | else: |
| 77 | dist.all_reduce(tensor, group=group) |
| 78 | |
| 79 | num_tokens_across_dp = tensor[0] |
| 80 | cg_mode_across_dp = tensor[1] |
| 81 | uniform_token_counts_across_dp = tensor[2] |
| 82 | max_query_lens_across_dp = tensor[3] |
| 83 | allow_ubatching_across_dp = tensor[4] |
| 84 | num_reqs_across_dp = tensor[5] |
| 85 | |
| 86 | # If ranks disagree on the uniform token count, or its 0 (means None) set to None |
| 87 | synced_uniform_token_count: int | None = int(uniform_token_counts_across_dp[0]) |
| 88 | if synced_uniform_token_count == 0 or not torch.all( |
| 89 | uniform_token_counts_across_dp == synced_uniform_token_count |
| 90 | ): |
| 91 | synced_uniform_token_count = None |
| 92 | |
| 93 | if torch.all(num_tokens_across_dp == 0).item(): |
| 94 | synced_desc = BatchExecutionDescriptor( |
| 95 | cg_mode=CUDAGraphMode.NONE, num_tokens=0, num_reqs=0 |
| 96 | ) |
| 97 | return synced_desc, None |
| 98 | |
| 99 | if torch.all(allow_ubatching_across_dp == 1): |
| 100 | # This rank voted too, so its caller passed a config. |
| 101 | assert parallel_config is not None |
| 102 | # A uniform decode only if every rank runs the same uniform query |
| 103 | # length, so every rank reaches the same answer here. |
| 104 | uniform_decode_across_dp = uniform_decode and torch.all( |
| 105 | uniform_token_counts_across_dp == (uniform_token_count or 0) |
| 106 | ) |
| 107 | if check_ubatch_thresholds( |
| 108 | parallel_config, |
| 109 | # Thresholds only grow with the token count, so holding the smallest |
| 110 | # rank to them holds every rank to them. |
| 111 | int(num_tokens_across_dp.min()), |
| 112 | uniform_decode=uniform_decode_across_dp, |
| 113 | ): |
| 114 | # Expert all-to-all requires every rank to split and pad equally. |
| 115 | # Empty microbatches run as dummy batches. |
| 116 | ubatch_num_tokens = int(num_tokens_across_dp.max()) |
| 117 | num_ubatches = get_num_ubatches(parallel_config) |
| 118 | ubatch_desc = None |
| 119 | if cudagraph_manager is not None: |
| 120 | # Match a FULL microbatched CUDA graph, falling back to an eager |
| 121 | # descriptor if no graph matches. |
| 122 | ubatch_desc = cudagraph_manager.dispatch( |
| 123 | num_reqs, |
| 124 | ubatch_num_tokens, |
| 125 | synced_uniform_token_count, |
| 126 | num_active_loras=num_active_loras, |
| 127 | num_ubatches=num_ubatches, |
| 128 | ) |
| 129 | if 2 * int(num_tokens_across_dp.min()) < ubatch_desc.num_tokens: |
| 130 | # If one rank has an empty second microbatch, run without |
| 131 | # CUDA graphs. |
| 132 | ubatch_desc = None |
| 133 | if ubatch_desc is None: |
| 134 | ubatch_desc = BatchExecutionDescriptor( |
| 135 | cg_mode=CUDAGraphMode.NONE, |
| 136 | num_tokens=ubatch_num_tokens, |
| 137 | num_reqs=num_reqs, |
| 138 | num_ubatches=num_ubatches, |
| 139 | ) |
| 140 | # Refresh the token count to include CUDA graph padding. |
| 141 | ubatch_num_tokens = ubatch_desc.num_tokens |
| 142 | num_reqs = int(num_reqs_across_dp.max()) |
| 143 | if ubatch_desc.cg_mode == CUDAGraphMode.FULL: |
| 144 | assert ubatch_desc.num_reqs is not None |
| 145 | num_reqs = ubatch_desc.num_reqs |
| 146 | return ubatch_desc, DPSyncState( |
| 147 | num_tokens_across_dp=torch.full_like( |
| 148 | num_tokens_across_dp, ubatch_num_tokens |
| 149 | ), |
| 150 | uniform_token_count=synced_uniform_token_count, |
| 151 | eager=ubatch_desc.cg_mode == CUDAGraphMode.NONE, |
| 152 | num_reqs=num_reqs, |
| 153 | ) |
| 154 | |
| 155 | synced_cg_mode = CUDAGraphMode(int(cg_mode_across_dp.min().item())) |
| 156 | |
| 157 | # If any rank wants to run eager, all ranks run eager |
| 158 | if synced_cg_mode == CUDAGraphMode.NONE: |
| 159 | return ( |
| 160 | BatchExecutionDescriptor( |
| 161 | cg_mode=CUDAGraphMode.NONE, |
| 162 | num_tokens=num_tokens, |
| 163 | num_reqs=num_reqs, |
| 164 | num_active_loras=desired_batch_desc.num_active_loras, |
| 165 | ), |
| 166 | DPSyncState( |
| 167 | num_tokens_across_dp=num_tokens_across_dp, |
| 168 | uniform_token_count=synced_uniform_token_count, |
| 169 | eager=True, |
| 170 | num_reqs=int(num_reqs_across_dp.max()), |
| 171 | ), |
| 172 | ) |
| 173 | |
| 174 | assert cudagraph_manager is not None, ( |
| 175 | "cudagraph_manager should only be None during profile run, " |
| 176 | "where synced_cg_mode must be NONE across all DP ranks" |
| 177 | ) |
| 178 | synced_num_tokens = int(num_tokens_across_dp.max().item()) |
| 179 | |
| 180 | # Varlen decode graphs are selected by the query-length bound, so ranks must agree |
| 181 | # on it or they pad to different token counts below. |
| 182 | synced_max_query_len: int | None = None |
| 183 | if bool(torch.all(max_query_lens_across_dp != -1).item()): |
| 184 | synced_max_query_len = int(max_query_lens_across_dp.max().item()) |
| 185 | |
| 186 | # Dispatch for the final synced values, use num_reqs instead of synced_num_reqs |
| 187 | # so we don't perform request padding for PIECEWISE graphs. |
| 188 | # num_active_loras is per-rank and doesn't need cross-rank agreement. |
| 189 | synced_desc = cudagraph_manager.dispatch( |
| 190 | num_reqs, |
| 191 | synced_num_tokens, |
| 192 | synced_uniform_token_count, |
| 193 | num_active_loras=num_active_loras, |
| 194 | max_query_len=synced_max_query_len, |
| 195 | ) |
| 196 | |
| 197 | # Update num_tokens_across_dp to reflect padded size. |
| 198 | num_tokens_across_dp[:] = synced_desc.num_tokens |
| 199 | |
| 200 | return synced_desc, DPSyncState( |
| 201 | num_tokens_across_dp=num_tokens_across_dp, |
| 202 | uniform_token_count=synced_uniform_token_count, |
| 203 | eager=False, |
| 204 | num_reqs=( |
| 205 | synced_desc.num_reqs |
| 206 | if synced_desc.cg_mode == CUDAGraphMode.FULL |
| 207 | and synced_desc.num_reqs is not None |
| 208 | else int(num_reqs_across_dp.max()) |
| 209 | ), |
| 210 | ) |
| 211 | |
| 212 | |
| 213 | def dispatch_cg_and_sync_dp( |
| 214 | cudagraph_manager: CudaGraphManager | None, |
| 215 | num_reqs: int, |
| 216 | num_tokens: int, |
| 217 | uniform_token_count: int | None, |
| 218 | dp_size: int, |
| 219 | dp_rank: int, |
| 220 | max_query_len: int | None = None, |
| 221 | need_eager: bool = False, |
| 222 | num_active_loras: int = 0, |
| 223 | parallel_config: ParallelConfig | None = None, |
| 224 | allow_ubatching: bool = False, |
| 225 | uniform_decode: bool = False, |
| 226 | dp_sync: DPSyncState | None = None, |
| 227 | ) -> tuple[BatchExecutionDescriptor, DPSyncState | None]: |
| 228 | """Pick a cudagraph descriptor for this batch, agreeing it across DP ranks. |
| 229 | |
| 230 | Runs a collective when dp_size > 1 so every rank dispatches to the same |
| 231 | shape. Pass `dp_sync` from a dispatch already made over this same batch (a |
| 232 | drafter's prefill runs the target's batch shape) to reuse that agreement |
| 233 | instead, with no collective. |
| 234 | |
| 235 | Args: |
| 236 | cudagraph_manager: Manager to dispatch against. May be None only when |
| 237 | `need_eager` is True (profile run). |
| 238 | num_reqs: Requests in this rank's batch. |
| 239 | num_tokens: Tokens in this rank's batch, already padded by the caller. |
| 240 | uniform_token_count: Per-request token count if this rank's batch is a |
| 241 | uniform decode, else None. `dp_sync.uniform_token_count` takes its |
| 242 | place when a sync is reused, since that one is agreed across ranks. |
| 243 | dp_size: Data-parallel world size. 1 skips all cross-rank work. |
| 244 | dp_rank: This rank's index in the DP group. |
| 245 | max_query_len: Upper bound on per-request query length, for selecting |
| 246 | varlen decode graphs. None, as for a batch with a prefill, keeps |
| 247 | the batch out of graphs that constrain it. |
| 248 | need_eager: Force `CUDAGraphMode.NONE` instead of dispatching. |
| 249 | num_active_loras: Active LoRA count for this rank. Does not need |
| 250 | cross-rank agreement; it never changes a bucket's token count. |
| 251 | parallel_config: Only needed to decide whether to microbatch, so |
| 252 | callers that never do (`allow_ubatching=False`) can leave it out. |
| 253 | allow_ubatching: Whether this rank's batch may be split into |
| 254 | microbatches. Agreed across ranks before it takes effect. |
| 255 | uniform_decode: Whether this rank's batch is a uniform decode, used |
| 256 | to pick the microbatching split. |
| 257 | dp_sync: Agreement from a prior dispatch over this same batch, to reuse. |
| 258 | Must come from a batch with this same padded `num_tokens` and the |
| 259 | same `uniform_token_count`; `num_reqs` may differ, as neither |
| 260 | depends on it. Passing a sync from a different batch is a caller |
| 261 | error and trips an assert. |
| 262 | |
| 263 | Returns: |
| 264 | (batch_desc, sync), where `sync` is this batch's agreement for a later |
| 265 | dispatch to reuse. It is None when `dp_size` is 1 or no rank has work. |
| 266 | |
| 267 | """ |
| 268 | reuse_eager = dp_sync is not None and dp_sync.eager |
| 269 | |
| 270 | if need_eager or reuse_eager: |
| 271 | batch_desc = BatchExecutionDescriptor( |
| 272 | cg_mode=CUDAGraphMode.NONE, |
| 273 | num_tokens=num_tokens, |
| 274 | num_reqs=num_reqs, |
| 275 | num_active_loras=num_active_loras, |
| 276 | ) |
| 277 | else: |
| 278 | assert cudagraph_manager is not None, ( |
| 279 | "cudagraph_manager should only be None during profile run, " |
| 280 | "where need_eager must be True" |
| 281 | ) |
| 282 | batch_desc = cudagraph_manager.dispatch( |
| 283 | num_reqs, |
| 284 | num_tokens, |
| 285 | dp_sync.uniform_token_count if dp_sync is not None else uniform_token_count, |
| 286 | num_active_loras=num_active_loras, |
| 287 | max_query_len=max_query_len, |
| 288 | ) |
| 289 | |
| 290 | if dp_size == 1: |
| 291 | return batch_desc, None |
| 292 | |
| 293 | if dp_sync is not None: |
| 294 | assert dp_sync.num_tokens_across_dp[dp_rank] == num_tokens, ( |
| 295 | "reusing a DP sync taken over a different batch" |
| 296 | ) |
| 297 | assert ( |
| 298 | dp_sync.uniform_token_count is None |
| 299 | or uniform_token_count == dp_sync.uniform_token_count |
| 300 | ), "reusing a DP sync taken over a different batch" |
| 301 | if not dp_sync.eager and batch_desc.num_tokens != num_tokens: |
| 302 | # Capture sizes can differ between managers, so this one may |
| 303 | # pad further. Every rank pads alike, so report what will run. |
| 304 | dp_sync = replace( |
| 305 | dp_sync, |
| 306 | num_tokens_across_dp=torch.full_like( |
| 307 | dp_sync.num_tokens_across_dp, batch_desc.num_tokens |
| 308 | ), |
| 309 | ) |
| 310 | return batch_desc, dp_sync |
| 311 | |
| 312 | return sync_cudagraph_and_dp_padding( |
| 313 | cudagraph_manager, |
| 314 | batch_desc, |
| 315 | num_tokens, |
| 316 | num_reqs, |
| 317 | uniform_token_count, |
| 318 | dp_size, |
| 319 | dp_rank, |
| 320 | max_query_len=max_query_len, |
| 321 | num_active_loras=num_active_loras, |
| 322 | parallel_config=parallel_config, |
| 323 | allow_ubatching=allow_ubatching, |
| 324 | uniform_decode=uniform_decode, |
| 325 | ) |