| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Unit tests for the breakable cudagraph primitives.""" |
| 4 | |
| 5 | from __future__ import annotations |
| 6 | |
| 7 | import threading |
| 8 | from contextlib import nullcontext |
| 9 | from unittest.mock import MagicMock, patch |
| 10 | |
| 11 | import pytest |
| 12 | import torch |
| 13 | |
| 14 | |
| 15 | @pytest.fixture(autouse=True) |
| 16 | def _enable_breakable_cudagraph(monkeypatch: pytest.MonkeyPatch): |
| 17 | """Enable breakable cudagraphs for this module's tests only. |
| 18 | |
| 19 | eager_break_during_capture reads the env at decoration time, which |
| 20 | happens inside the test bodies, so a per-test fixture suffices. |
| 21 | monkeypatch restores the env so other test files running in the same |
| 22 | pytest process are unaffected (a module-level os.environ assignment |
| 23 | used to leak into test_cudagraph_dispatch.py and break it). |
| 24 | """ |
| 25 | import vllm.envs as envs |
| 26 | |
| 27 | monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "1") |
| 28 | envs.disable_envs_cache() |
| 29 | |
| 30 | |
| 31 | def test_piecewise_capture_builds_fresh_metadata_for_both_passes(): |
| 32 | from vllm.config import CUDAGraphMode |
| 33 | from vllm.v1.worker.gpu.cudagraph_utils import ( |
| 34 | BatchExecutionDescriptor, |
| 35 | CudaGraphManager, |
| 36 | ) |
| 37 | |
| 38 | manager = CudaGraphManager.__new__(CudaGraphManager) |
| 39 | desc = BatchExecutionDescriptor(CUDAGraphMode.PIECEWISE, 8, None) |
| 40 | manager.device = torch.device("cpu") |
| 41 | manager._capture_descs = {CUDAGraphMode.PIECEWISE: [desc]} |
| 42 | manager._graphs_captured = False |
| 43 | manager.use_breakable_cg = True |
| 44 | manager.ubatch_runner = None |
| 45 | |
| 46 | create_calls = [] |
| 47 | forward_calls = [] |
| 48 | |
| 49 | def create_forward_fn(desc_arg, warmup): |
| 50 | assert desc_arg == desc |
| 51 | metadata = {"layer": object()} |
| 52 | create_calls.append((warmup, metadata)) |
| 53 | |
| 54 | def forward_fn(cg_mode): |
| 55 | assert metadata |
| 56 | forward_calls.append((warmup, cg_mode, metadata)) |
| 57 | |
| 58 | return forward_fn |
| 59 | |
| 60 | with ( |
| 61 | patch( |
| 62 | "vllm.v1.worker.gpu.cudagraph_utils.graph_capture", |
| 63 | return_value=nullcontext(), |
| 64 | ), |
| 65 | patch( |
| 66 | "vllm.v1.worker.gpu.cudagraph_utils.is_global_first_rank", |
| 67 | return_value=False, |
| 68 | ), |
| 69 | ): |
| 70 | manager.capture(create_forward_fn) |
| 71 | |
| 72 | assert [warmup for warmup, _ in create_calls] == [True, False] |
| 73 | assert [mode for _, mode, _ in forward_calls] == [ |
| 74 | CUDAGraphMode.NONE, |
| 75 | CUDAGraphMode.PIECEWISE, |
| 76 | ] |
| 77 | assert create_calls[0][1] is not create_calls[1][1] |
| 78 | |
| 79 | |
| 80 | @pytest.fixture(autouse=True) |
| 81 | def _reset_breakable_tls(): |
| 82 | """Defensively clear thread-local capture state between tests so a |
| 83 | failure in one test can't leak "nested capture" errors into the next.""" |
| 84 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 85 | |
| 86 | BreakableCUDAGraphCapture._tls.active = None |
| 87 | yield |
| 88 | BreakableCUDAGraphCapture._tls.active = None |
| 89 | |
| 90 | |
| 91 | @pytest.fixture |
| 92 | def cuda_capture_stream(): |
| 93 | """A non-default CUDA stream suitable for cudagraph capture. |
| 94 | |
| 95 | ``CUDAGraph.capture_begin`` refuses to capture from the default |
| 96 | stream, so all capture-using tests need to run under |
| 97 | ``torch.cuda.stream(...)`` for a separate stream. |
| 98 | """ |
| 99 | if not torch.cuda.is_available(): |
| 100 | pytest.skip("CUDA required") |
| 101 | from vllm.utils.torch_utils import _current_stream_tls |
| 102 | |
| 103 | prev_stream = getattr(_current_stream_tls, "value", None) |
| 104 | stream = torch.cuda.Stream() |
| 105 | with torch.cuda.stream(stream): |
| 106 | yield stream |
| 107 | torch.cuda.current_stream().wait_stream(stream) |
| 108 | # Exiting torch.cuda.stream() records the default stream in vllm's |
| 109 | # patched set_stream cache. A later CUDAGraphWrapper capture in the same |
| 110 | # process would then run on the default stream, which cannot capture |
| 111 | # (this broke test_cudagraph_dispatch.py when run in-process after this |
| 112 | # file). Restore the pre-fixture value so this module leaves no trace. |
| 113 | _current_stream_tls.value = prev_stream |
| 114 | |
| 115 | |
| 116 | # --------------------------------------------------------------------------- |
| 117 | # eager_break_during_capture: outside capture |
| 118 | # --------------------------------------------------------------------------- |
| 119 | |
| 120 | |
| 121 | def test_decorator_passthrough_outside_capture(): |
| 122 | from vllm.compilation.breakable_cudagraph import eager_break_during_capture |
| 123 | |
| 124 | calls = [] |
| 125 | |
| 126 | @eager_break_during_capture |
| 127 | def f(x): |
| 128 | calls.append(x) |
| 129 | return x * 2 |
| 130 | |
| 131 | assert f(3) == 6 |
| 132 | assert calls == [3] |
| 133 | |
| 134 | |
| 135 | # --------------------------------------------------------------------------- |
| 136 | # BreakableCUDAGraphCapture: thread-local + nested rejection |
| 137 | # --------------------------------------------------------------------------- |
| 138 | |
| 139 | |
| 140 | def test_current_is_none_when_inactive(): |
| 141 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 142 | |
| 143 | assert BreakableCUDAGraphCapture.current() is None |
| 144 | assert BreakableCUDAGraphCapture.is_active() is False |
| 145 | |
| 146 | |
| 147 | def test_thread_local_active_during_context(cuda_capture_stream): |
| 148 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 149 | |
| 150 | cap = BreakableCUDAGraphCapture() |
| 151 | with cap: |
| 152 | assert BreakableCUDAGraphCapture.current() is cap |
| 153 | assert BreakableCUDAGraphCapture.is_active() is True |
| 154 | assert BreakableCUDAGraphCapture.current() is None |
| 155 | |
| 156 | |
| 157 | def test_nested_capture_raises(cuda_capture_stream): |
| 158 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 159 | |
| 160 | outer = BreakableCUDAGraphCapture() |
| 161 | inner = BreakableCUDAGraphCapture() |
| 162 | with outer, pytest.raises(RuntimeError, match="Nested.*not supported"), inner: |
| 163 | pass |
| 164 | |
| 165 | |
| 166 | def test_active_state_isolated_across_threads(cuda_capture_stream): |
| 167 | """Verify the thread-local 'active capture' slot is per-thread. |
| 168 | |
| 169 | We don't run concurrent captures here -- CUDA only supports one |
| 170 | in-flight capture per stream and we keep tests cheap. We just check |
| 171 | that the worker thread sees its own slot as None while the main |
| 172 | thread has a capture active. |
| 173 | """ |
| 174 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 175 | |
| 176 | worker_view: dict[str, BreakableCUDAGraphCapture | None] = {} |
| 177 | |
| 178 | def worker(): |
| 179 | worker_view["state"] = BreakableCUDAGraphCapture.current() |
| 180 | |
| 181 | main_cap = BreakableCUDAGraphCapture() |
| 182 | with main_cap: |
| 183 | # Main thread has a live capture. |
| 184 | assert BreakableCUDAGraphCapture.current() is main_cap |
| 185 | t = threading.Thread(target=worker) |
| 186 | t.start() |
| 187 | t.join() |
| 188 | |
| 189 | # Worker thread saw None -- thread-local separation works. |
| 190 | assert worker_view["state"] is None |
| 191 | # Main thread's slot is cleared on exit. |
| 192 | assert BreakableCUDAGraphCapture.current() is None |
| 193 | |
| 194 | |
| 195 | # --------------------------------------------------------------------------- |
| 196 | # Segment list construction |
| 197 | # --------------------------------------------------------------------------- |
| 198 | |
| 199 | |
| 200 | def test_capture_with_no_eager_break_records_one_graph(cuda_capture_stream): |
| 201 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 202 | |
| 203 | x = torch.zeros(4, device="cuda") |
| 204 | cap = BreakableCUDAGraphCapture() |
| 205 | with cap: |
| 206 | x.add_(1.0) |
| 207 | assert len(cap.segments) == 1 |
| 208 | assert cap.num_graphs == 1 |
| 209 | assert cap.num_eager_breaks == 0 |
| 210 | |
| 211 | |
| 212 | def test_add_eager_creates_alternating_graph_eager_graph(cuda_capture_stream): |
| 213 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 214 | |
| 215 | x = torch.zeros(4, device="cuda") |
| 216 | counter = {"eager_calls": 0} |
| 217 | |
| 218 | def eager_step(): |
| 219 | counter["eager_calls"] += 1 |
| 220 | x.add_(10.0) |
| 221 | |
| 222 | cap = BreakableCUDAGraphCapture() |
| 223 | with cap: |
| 224 | x.add_(1.0) |
| 225 | cap.add_eager(eager_step) |
| 226 | x.add_(1.0) |
| 227 | cap.add_eager(eager_step) |
| 228 | x.add_(1.0) |
| 229 | # 3 graph segments + 2 eager segments, interleaved as G E G E G. |
| 230 | assert len(cap.segments) == 5 |
| 231 | assert cap.num_graphs == 3 |
| 232 | assert cap.num_eager_breaks == 2 |
| 233 | # Eager fn is stored as-is in the segment list, so we can confirm |
| 234 | # the alternation pattern by identity check. |
| 235 | assert cap.segments[1] is eager_step |
| 236 | assert cap.segments[3] is eager_step |
| 237 | assert counter["eager_calls"] == 2 # only the in-capture invocation |
| 238 | |
| 239 | |
| 240 | # --------------------------------------------------------------------------- |
| 241 | # Capture vs eager numerical equivalence |
| 242 | # --------------------------------------------------------------------------- |
| 243 | |
| 244 | |
| 245 | def test_capture_replay_matches_eager_simple(cuda_capture_stream): |
| 246 | """Verify that replay reproduces the same end-state as a single eager |
| 247 | forward, with an eager break in the middle. |
| 248 | |
| 249 | Note: during capture, the *captured* kernels are recorded but NOT |
| 250 | executed (that's CUDA-graph semantics). Only the eager segments |
| 251 | actually mutate state at capture time. So we check correctness after |
| 252 | ``replay()``, not after ``with cap:`` exits. |
| 253 | """ |
| 254 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 255 | |
| 256 | x = torch.zeros(8, device="cuda") |
| 257 | log: list[str] = [] |
| 258 | |
| 259 | def eager_break_op(): |
| 260 | x.mul_(2.0) |
| 261 | log.append("eager") |
| 262 | |
| 263 | cap = BreakableCUDAGraphCapture() |
| 264 | with cap: |
| 265 | x.add_(1.0) # recorded into graph[0] |
| 266 | cap.add_eager(eager_break_op) # runs eagerly: x *= 2 |
| 267 | x.add_(5.0) # recorded into graph[1] |
| 268 | |
| 269 | # Capture-time: graph kernels were recorded only; eager segment ran |
| 270 | # once on x == 0, leaving x == 0. |
| 271 | torch.accelerator.synchronize() |
| 272 | assert torch.equal(x, torch.zeros(8, device="cuda")) |
| 273 | assert log == ["eager"] |
| 274 | |
| 275 | # Replay with a fresh input: 10 -> 11 -> 22 -> 27. |
| 276 | x.fill_(10.0) |
| 277 | cap.replay() |
| 278 | torch.accelerator.synchronize() |
| 279 | assert torch.equal(x, torch.full((8,), 27.0, device="cuda")) |
| 280 | assert log == ["eager", "eager"] |
| 281 | |
| 282 | # Replay again with another input: 100 -> 101 -> 202 -> 207. |
| 283 | x.fill_(100.0) |
| 284 | cap.replay() |
| 285 | torch.accelerator.synchronize() |
| 286 | assert torch.equal(x, torch.full((8,), 207.0, device="cuda")) |
| 287 | assert log == ["eager", "eager", "eager"] |
| 288 | |
| 289 | |
| 290 | def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream): |
| 291 | """Verify @eager_break_during_capture correctly routes through |
| 292 | add_eager when inside a capture context, and runs straight through |
| 293 | when there's no active capture.""" |
| 294 | from vllm.compilation.breakable_cudagraph import ( |
| 295 | BreakableCUDAGraphCapture, |
| 296 | eager_break_during_capture, |
| 297 | ) |
| 298 | |
| 299 | @eager_break_during_capture |
| 300 | def attention_like(t: torch.Tensor) -> None: |
| 301 | # In-place double; stands in for "real" attention work. |
| 302 | t.mul_(2.0) |
| 303 | |
| 304 | x = torch.zeros(4, device="cuda") |
| 305 | |
| 306 | # Outside capture: decorator should just call through. |
| 307 | x.fill_(3.0) |
| 308 | attention_like(x) |
| 309 | torch.accelerator.synchronize() |
| 310 | assert torch.equal(x, torch.full((4,), 6.0, device="cuda")) |
| 311 | |
| 312 | # Inside capture: decorator should split the graph. Only the eager |
| 313 | # segment actually mutates state during capture. |
| 314 | x.fill_(0.0) |
| 315 | cap = BreakableCUDAGraphCapture() |
| 316 | with cap: |
| 317 | x.add_(5.0) # recorded |
| 318 | attention_like(x) # eager: x *= 2 (on x == 0, no-op) |
| 319 | x.add_(1.0) # recorded |
| 320 | torch.accelerator.synchronize() |
| 321 | assert torch.equal(x, torch.zeros(4, device="cuda")) |
| 322 | # 2 graph segments + 1 eager segment, ordered G E G; the arithmetic |
| 323 | # equivalence check below verifies the ordering. |
| 324 | assert len(cap.segments) == 3 |
| 325 | assert cap.num_graphs == 2 |
| 326 | assert cap.num_eager_breaks == 1 |
| 327 | |
| 328 | # Replay: 2 -> 7 -> 14 -> 15. |
| 329 | x.fill_(2.0) |
| 330 | cap.replay() |
| 331 | torch.accelerator.synchronize() |
| 332 | assert torch.equal(x, torch.full((4,), 15.0, device="cuda")) |
| 333 | |
| 334 | |
| 335 | @pytest.mark.parametrize("keyword", [False, True]) |
| 336 | def test_quantized_activation_replay_does_not_retain_tensors( |
| 337 | cuda_capture_stream, keyword |
| 338 | ): |
| 339 | """Release the producer tensors while preserving both buffers on replay.""" |
| 340 | import gc |
| 341 | import weakref |
| 342 | |
| 343 | from vllm.compilation.breakable_cudagraph import ( |
| 344 | BreakableCUDAGraphCapture, |
| 345 | eager_break_during_capture, |
| 346 | ) |
| 347 | from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation |
| 348 | from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic |
| 349 | |
| 350 | @eager_break_during_capture |
| 351 | def consume(x, data_out, scale_out): |
| 352 | data_out.copy_(x.data) |
| 353 | scale_out.copy_(x.scale) |
| 354 | |
| 355 | source = torch.ones((128, 128), device="cuda", dtype=torch.bfloat16) |
| 356 | data_out = source.to(torch.float8_e4m3fn) |
| 357 | scale_out = source[:, ::32].to(torch.uint8) |
| 358 | output = torch.empty_like(source) |
| 359 | cap = BreakableCUDAGraphCapture() |
| 360 | with cap: |
| 361 | data = source.to(torch.float8_e4m3fn) |
| 362 | scale = source[:, ::32].to(torch.uint8) |
| 363 | activation = QuantizedActivation( |
| 364 | data, scale, source.dtype, source.shape, kMxfp8Dynamic |
| 365 | ) |
| 366 | data_ref, scale_ref = weakref.ref(data), weakref.ref(scale) |
| 367 | if keyword: |
| 368 | consume(x=activation, data_out=data_out, scale_out=scale_out) |
| 369 | else: |
| 370 | consume(activation, data_out, scale_out) |
| 371 | output.copy_(data_out) |
| 372 | |
| 373 | del data, scale, activation |
| 374 | gc.collect() |
| 375 | assert data_ref() is None |
| 376 | assert scale_ref() is None |
| 377 | |
| 378 | for value in (1, 2, 3): |
| 379 | source.fill_(value) |
| 380 | cap.replay() |
| 381 | cuda_capture_stream.synchronize() |
| 382 | torch.testing.assert_close(output, source, rtol=0, atol=0) |
| 383 | torch.testing.assert_close(scale_out, source[:, ::32].to(torch.uint8)) |
| 384 | |
| 385 | |
| 386 | def test_eager_attention_inside_multistream_overlap(cuda_capture_stream): |
| 387 | """Handle an eager attention break inside a multi-stream overlap region.""" |
| 388 | from vllm.compilation.breakable_cudagraph import ( |
| 389 | BreakableCUDAGraphCapture, |
| 390 | eager_break_during_capture, |
| 391 | ) |
| 392 | from vllm.utils.multi_stream_utils import maybe_execute_in_parallel |
| 393 | |
| 394 | x = torch.zeros(1024, device="cuda") |
| 395 | output = torch.empty_like(x) |
| 396 | aux_stream = torch.cuda.Stream() |
| 397 | main_event = torch.cuda.Event() |
| 398 | aux_event = torch.cuda.Event() |
| 399 | |
| 400 | @eager_break_during_capture |
| 401 | def attention(query: torch.Tensor, out: torch.Tensor) -> None: |
| 402 | torch.add(query, 3.0, out=out) |
| 403 | |
| 404 | def attention_frontend() -> torch.Tensor: |
| 405 | query = x * 2.0 |
| 406 | attention_output = torch.empty_like(x) |
| 407 | attention(query, attention_output) |
| 408 | return attention_output |
| 409 | |
| 410 | cap = BreakableCUDAGraphCapture() |
| 411 | with cap: |
| 412 | attention_output, gate = maybe_execute_in_parallel( |
| 413 | attention_frontend, |
| 414 | lambda: x * 5.0, |
| 415 | main_event, |
| 416 | aux_event, |
| 417 | aux_stream, |
| 418 | ) |
| 419 | torch.add(attention_output, gate, out=output) |
| 420 | |
| 421 | assert cap.num_graphs == 2 |
| 422 | assert cap.num_eager_breaks == 1 |
| 423 | |
| 424 | for value in (1.0, 7.0, 19.0): |
| 425 | x.fill_(value) |
| 426 | cap.replay() |
| 427 | cuda_capture_stream.synchronize() |
| 428 | torch.testing.assert_close( |
| 429 | output, |
| 430 | torch.full_like(output, value * 7.0 + 3.0), |
| 431 | ) |
| 432 | |
| 433 | |
| 434 | # --------------------------------------------------------------------------- |
| 435 | # Replay ordering |
| 436 | # --------------------------------------------------------------------------- |
| 437 | |
| 438 | |
| 439 | def test_replay_invokes_eager_segments_in_order(cuda_capture_stream): |
| 440 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 441 | |
| 442 | log: list[str] = [] |
| 443 | x = torch.zeros(1, device="cuda") |
| 444 | |
| 445 | def make_eager(name): |
| 446 | def step(): |
| 447 | log.append(name) |
| 448 | x.add_(1.0) |
| 449 | |
| 450 | return step |
| 451 | |
| 452 | cap = BreakableCUDAGraphCapture() |
| 453 | with cap: |
| 454 | x.add_(1.0) |
| 455 | cap.add_eager(make_eager("A")) |
| 456 | x.add_(1.0) |
| 457 | cap.add_eager(make_eager("B")) |
| 458 | x.add_(1.0) |
| 459 | cap.add_eager(make_eager("C")) |
| 460 | x.add_(1.0) |
| 461 | |
| 462 | # Capture-time invocation order |
| 463 | assert log == ["A", "B", "C"] |
| 464 | |
| 465 | log.clear() |
| 466 | cap.replay() |
| 467 | torch.accelerator.synchronize() |
| 468 | assert log == ["A", "B", "C"] |
| 469 | |
| 470 | |
| 471 | # --------------------------------------------------------------------------- |
| 472 | # Capture cleanup releases thread-local even if body raises |
| 473 | # --------------------------------------------------------------------------- |
| 474 | |
| 475 | |
| 476 | def test_exception_in_body_clears_active(cuda_capture_stream): |
| 477 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture |
| 478 | |
| 479 | cap = BreakableCUDAGraphCapture() |
| 480 | with pytest.raises(RuntimeError, match="boom"), cap: |
| 481 | raise RuntimeError("boom") |
| 482 | |
| 483 | # active must be reset even after an exception inside the body |
| 484 | assert BreakableCUDAGraphCapture.current() is None |
| 485 | |
| 486 | |
| 487 | # --------------------------------------------------------------------------- |
| 488 | # Nested decorated ops: inner op must not trigger a recursive eager break |
| 489 | # --------------------------------------------------------------------------- |
| 490 | |
| 491 | |
| 492 | def test_nested_decorated_op_runs_inline(cuda_capture_stream): |
| 493 | """A decorated op invoked from inside another decorated op's eager |
| 494 | body must execute inline -- starting a second eager break mid-flight |
| 495 | corrupts the segment state and explodes ``_begin_segment``'s assert. |
| 496 | |
| 497 | This mirrors the deepseek_v4_attention case where the outer attention |
| 498 | op's impl internally dispatches sparse_attn_indexer (also decorated). |
| 499 | """ |
| 500 | from vllm.compilation.breakable_cudagraph import ( |
| 501 | BreakableCUDAGraphCapture, |
| 502 | eager_break_during_capture, |
| 503 | ) |
| 504 | |
| 505 | x = torch.zeros(4, device="cuda") |
| 506 | inner_calls = 0 |
| 507 | |
| 508 | @eager_break_during_capture |
| 509 | def inner_op(t: torch.Tensor) -> None: |
| 510 | nonlocal inner_calls |
| 511 | inner_calls += 1 |
| 512 | t.add_(1.0) |
| 513 | |
| 514 | @eager_break_during_capture |
| 515 | def outer_op(t: torch.Tensor) -> None: |
| 516 | # outer body calls another decorated op -- this is the case that |
| 517 | # used to assert in _begin_segment. |
| 518 | inner_op(t) |
| 519 | t.add_(10.0) |
| 520 | |
| 521 | cap = BreakableCUDAGraphCapture() |
| 522 | with cap: |
| 523 | x.add_(2.0) # recorded in graph[0] |
| 524 | outer_op(x) # one eager break, inner runs inline |
| 525 | x.add_(100.0) # recorded in graph[1] |
| 526 | |
| 527 | # Exactly one eager break (the outer); inner must NOT add a second. |
| 528 | assert cap.num_graphs == 2 |
| 529 | assert cap.num_eager_breaks == 1 |
| 530 | assert inner_calls == 1 # only the capture-time invocation |
| 531 | |
| 532 | x.fill_(0.0) |
| 533 | cap.replay() |
| 534 | torch.accelerator.synchronize() |
| 535 | # 0 -> +2 -> +1 (inner) -> +10 (outer) -> +100 = 113 |
| 536 | assert torch.equal(x, torch.full((4,), 113.0, device="cuda")) |
| 537 | assert inner_calls == 2 # replay invokes the outer's lambda again |
| 538 | |
| 539 | |
| 540 | # --------------------------------------------------------------------------- |
| 541 | # BreakableCUDAGraphWrapper: runtime_mode dispatch gating |
| 542 | # --------------------------------------------------------------------------- |
| 543 | |
| 544 | |
| 545 | def _mock_vllm_config(): |
| 546 | config = MagicMock() |
| 547 | config.parallel_config.data_parallel_size = 1 |
| 548 | config.parallel_config.use_sequence_parallel_moe = False |
| 549 | config.parallel_config.is_moe_model = False |
| 550 | return config |
| 551 | |
| 552 | |
| 553 | def test_wrapper_falls_through_on_runtime_mode_mismatch(): |
| 554 | """A PIECEWISE-scoped wrapper runs other modes eagerly, capturing |
| 555 | nothing, so an outer CUDAGraphWrapper can own them.""" |
| 556 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper |
| 557 | from vllm.config import CUDAGraphMode |
| 558 | from vllm.forward_context import BatchDescriptor, set_forward_context |
| 559 | |
| 560 | calls = [] |
| 561 | |
| 562 | def model(x): |
| 563 | calls.append(x) |
| 564 | return x + 1 |
| 565 | |
| 566 | config = _mock_vllm_config() |
| 567 | wrapper = BreakableCUDAGraphWrapper( |
| 568 | model, config, runtime_mode=CUDAGraphMode.PIECEWISE |
| 569 | ) |
| 570 | desc = BatchDescriptor(num_tokens=4) |
| 571 | |
| 572 | with set_forward_context( |
| 573 | None, |
| 574 | config, |
| 575 | cudagraph_runtime_mode=CUDAGraphMode.FULL, |
| 576 | batch_descriptor=desc, |
| 577 | ): |
| 578 | assert wrapper(1) == 2 |
| 579 | |
| 580 | assert calls == [1] |
| 581 | assert not wrapper.entries |
| 582 | |
| 583 | |
| 584 | def test_wrapper_captures_on_runtime_mode_match(): |
| 585 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper |
| 586 | from vllm.config import CUDAGraphMode |
| 587 | from vllm.forward_context import BatchDescriptor, set_forward_context |
| 588 | |
| 589 | calls = [] |
| 590 | |
| 591 | def model(x): |
| 592 | calls.append(x) |
| 593 | return x + 1 |
| 594 | |
| 595 | config = _mock_vllm_config() |
| 596 | wrapper = BreakableCUDAGraphWrapper( |
| 597 | model, config, runtime_mode=CUDAGraphMode.PIECEWISE |
| 598 | ) |
| 599 | desc = BatchDescriptor(num_tokens=4) |
| 600 | |
| 601 | sentinel = object() |
| 602 | with ( |
| 603 | patch.object( |
| 604 | BreakableCUDAGraphWrapper, "_capture", return_value=sentinel |
| 605 | ) as mock_capture, |
| 606 | set_forward_context( |
| 607 | None, |
| 608 | config, |
| 609 | cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE, |
| 610 | batch_descriptor=desc, |
| 611 | ), |
| 612 | ): |
| 613 | assert wrapper(1) is sentinel |
| 614 | |
| 615 | mock_capture.assert_called_once() |
| 616 | assert calls == [] |
| 617 | assert desc in wrapper.entries |
| 618 | |
| 619 | |
| 620 | def test_wrapper_default_matches_any_non_none_mode(): |
| 621 | """runtime_mode=None matches any non-NONE mode (MRV2 relies on this).""" |
| 622 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper |
| 623 | from vllm.config import CUDAGraphMode |
| 624 | from vllm.forward_context import BatchDescriptor, set_forward_context |
| 625 | |
| 626 | def model(x): |
| 627 | return x + 1 |
| 628 | |
| 629 | config = _mock_vllm_config() |
| 630 | wrapper = BreakableCUDAGraphWrapper(model, config) |
| 631 | desc = BatchDescriptor(num_tokens=4) |
| 632 | |
| 633 | sentinel = object() |
| 634 | with ( |
| 635 | patch.object( |
| 636 | BreakableCUDAGraphWrapper, "_capture", return_value=sentinel |
| 637 | ) as mock_capture, |
| 638 | set_forward_context( |
| 639 | None, |
| 640 | config, |
| 641 | cudagraph_runtime_mode=CUDAGraphMode.FULL, |
| 642 | batch_descriptor=desc, |
| 643 | ), |
| 644 | ): |
| 645 | assert wrapper(1) is sentinel |
| 646 | |
| 647 | mock_capture.assert_called_once() |
| 648 | assert desc in wrapper.entries |
| 649 | |
| 650 | |
| 651 | def test_unwrap_recurses_through_nested_wrappers(): |
| 652 | """unwrap() reaches the raw model through nested wrappers.""" |
| 653 | from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper |
| 654 | from vllm.compilation.cuda_graph import CUDAGraphWrapper |
| 655 | from vllm.config import CUDAGraphMode |
| 656 | |
| 657 | def model(x): |
| 658 | return x + 1 |
| 659 | |
| 660 | config = _mock_vllm_config() |
| 661 | inner = BreakableCUDAGraphWrapper( |
| 662 | model, config, runtime_mode=CUDAGraphMode.PIECEWISE |
| 663 | ) |
| 664 | outer = CUDAGraphWrapper(inner, config, runtime_mode=CUDAGraphMode.FULL) |
| 665 | |
| 666 | assert inner.unwrap() is model |
| 667 | assert outer.unwrap() is model |