| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Warm Mamba-style Triton kernels shared across GDN / Mamba / KDA models.""" |
| 4 | |
| 5 | from typing import TYPE_CHECKING |
| 6 | |
| 7 | import torch |
| 8 | |
| 9 | from vllm.logger import init_logger |
| 10 | |
| 11 | if TYPE_CHECKING: |
| 12 | from vllm.v1.worker.gpu_model_runner import GPUModelRunner |
| 13 | |
| 14 | logger = init_logger(__name__) |
| 15 | |
| 16 | |
| 17 | def _has_mamba_style_cache(runner: "GPUModelRunner") -> bool: |
| 18 | from vllm.v1.kv_cache_interface import MambaSpec, UniformTypeKVCacheSpecs |
| 19 | |
| 20 | for group in runner.kv_cache_config.kv_cache_groups: |
| 21 | spec = group.kv_cache_spec |
| 22 | if isinstance(spec, UniformTypeKVCacheSpecs): |
| 23 | spec = spec.first_spec |
| 24 | if isinstance(spec, MambaSpec): |
| 25 | return True |
| 26 | return False |
| 27 | |
| 28 | |
| 29 | def _warm_batch_memcpy_kernel(device: torch.device) -> None: |
| 30 | """Warm the Mamba prefix-cache state copy specialization.""" |
| 31 | from vllm.v1.worker.mamba_utils import batch_memcpy |
| 32 | |
| 33 | src = torch.empty(1024, dtype=torch.uint8, device=device) |
| 34 | dst = torch.empty_like(src) |
| 35 | batch_memcpy( |
| 36 | torch.tensor([src.data_ptr()], dtype=torch.uint64, device=device), |
| 37 | torch.tensor([dst.data_ptr()], dtype=torch.uint64, device=device), |
| 38 | # Keep this int32: Triton's compile key includes pointer element dtypes, |
| 39 | # and the production prefix-cache path passes an int32 sizes tensor. |
| 40 | torch.tensor([src.numel()], dtype=torch.int32, device=device), |
| 41 | ) |
| 42 | logger.info("Warmed Mamba batch_memcpy_kernel.") |
| 43 | |
| 44 | |
| 45 | @torch.inference_mode() |
| 46 | def mamba_triton_warmup(runner: "GPUModelRunner") -> None: |
| 47 | """Warm prefix-cache memcpy for every Mamba-style KV cache group.""" |
| 48 | device = runner.device |
| 49 | if device.type != "cuda": |
| 50 | return |
| 51 | if not _has_mamba_style_cache(runner): |
| 52 | return |
| 53 | _warm_batch_memcpy_kernel(device) |