| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Custom ops for prefetch offloader torch.compile + CUDA graph compatibility. |
| 4 | |
| 5 | These ops use mutates_args to create data dependencies that prevent |
| 6 | the compiler from reordering prefetch/sync operations. |
| 7 | """ |
| 8 | |
| 9 | from __future__ import annotations |
| 10 | |
| 11 | import torch |
| 12 | |
| 13 | from vllm.model_executor.offloader.base import get_offloader |
| 14 | from vllm.utils.torch_utils import direct_register_custom_op |
| 15 | |
| 16 | # --- wait_prefetch op --- |
| 17 | |
| 18 | |
| 19 | def _wait_prefetch_impl( |
| 20 | input_tensor: torch.Tensor, |
| 21 | layer_idx: int, |
| 22 | ) -> None: |
| 23 | """Wait for prefetch of layer_idx to complete. |
| 24 | |
| 25 | Synchronizes the compute stream with the copy stream to ensure |
| 26 | the prefetched weights are ready for use. |
| 27 | |
| 28 | Args: |
| 29 | input_tensor: Input to the layer (e.g., hidden_states) - declared |
| 30 | as mutated to create data dependency for torch.compile. |
| 31 | layer_idx: Index of the layer to wait for. |
| 32 | |
| 33 | """ |
| 34 | get_offloader()._wait_for_layer(layer_idx) |
| 35 | |
| 36 | |
| 37 | # --- start_prefetch op --- |
| 38 | |
| 39 | |
| 40 | def _start_prefetch_impl( |
| 41 | output_tensor: torch.Tensor, |
| 42 | layer_idx: int, |
| 43 | ) -> None: |
| 44 | """Start async prefetch of layer_idx weights. |
| 45 | |
| 46 | Initiates H2D copy on the copy stream for the specified layer. |
| 47 | |
| 48 | Args: |
| 49 | output_tensor: Output from forward - declared as mutated to |
| 50 | prevent torch.compile from reordering this op before the |
| 51 | computation that produces output_tensor. |
| 52 | layer_idx: Index of the layer to prefetch. |
| 53 | |
| 54 | """ |
| 55 | get_offloader()._start_prefetch(layer_idx) |
| 56 | |
| 57 | |
| 58 | def register_prefetch_offloader_ops() -> None: |
| 59 | """Register custom ops for prefetch offloader. |
| 60 | |
| 61 | Must be called before the ops are used. This is typically done |
| 62 | at module import time. |
| 63 | """ |
| 64 | direct_register_custom_op( |
| 65 | op_name="wait_prefetch", |
| 66 | op_func=_wait_prefetch_impl, |
| 67 | mutates_args=["input_tensor"], |
| 68 | ) |
| 69 | |
| 70 | direct_register_custom_op( |
| 71 | op_name="start_prefetch", |
| 72 | op_func=_start_prefetch_impl, |
| 73 | mutates_args=["output_tensor"], |
| 74 | ) |
| 75 | |
| 76 | |
| 77 | # Register ops at module import time |
| 78 | register_prefetch_offloader_ops() |