| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | |
| 5 | import torch |
| 6 | |
| 7 | import vllm.envs as envs |
| 8 | from vllm import _custom_ops as ops |
| 9 | from vllm.platforms import current_platform |
| 10 | from vllm.utils.platform_utils import num_compute_units |
| 11 | from vllm.utils.torch_utils import direct_register_custom_op |
| 12 | |
| 13 | from .ScaledMMLinearKernel import ( |
| 14 | FP8ScaledMMLinearKernel, |
| 15 | FP8ScaledMMLinearLayerConfig, |
| 16 | ) |
| 17 | |
| 18 | |
| 19 | def rocm_per_tensor_float_w8a8_scaled_mm_impl( |
| 20 | A: torch.Tensor, |
| 21 | B: torch.Tensor, |
| 22 | out_dtype: torch.dtype, |
| 23 | As: torch.Tensor, |
| 24 | Bs: torch.Tensor, |
| 25 | bias: torch.Tensor, |
| 26 | ) -> torch.Tensor: |
| 27 | if ( |
| 28 | A.shape[0] <= 4 |
| 29 | and B.shape[0] % 16 == 0 # M TODO: needed? |
| 30 | and B.shape[1] % 16 == 0 # K |
| 31 | and ((bias is None) or (bias.dtype == out_dtype)) |
| 32 | ): |
| 33 | output = ops.wvSplitKQ( |
| 34 | B.t(), |
| 35 | A, |
| 36 | out_dtype, |
| 37 | As, |
| 38 | Bs, |
| 39 | num_compute_units(), |
| 40 | bias, |
| 41 | ) |
| 42 | # Fallback |
| 43 | else: |
| 44 | output = torch._scaled_mm( |
| 45 | A, |
| 46 | B, |
| 47 | out_dtype=out_dtype, |
| 48 | scale_a=As, |
| 49 | scale_b=Bs, |
| 50 | bias=bias, |
| 51 | ) |
| 52 | return output |
| 53 | |
| 54 | |
| 55 | def rocm_per_tensor_float_w8a8_scaled_mm_fake( |
| 56 | A: torch.Tensor, |
| 57 | B: torch.Tensor, |
| 58 | out_dtype: torch.dtype, |
| 59 | As: torch.Tensor, |
| 60 | Bs: torch.Tensor, |
| 61 | bias: torch.Tensor, |
| 62 | ) -> torch.Tensor: |
| 63 | return A.new_empty((*A.shape[:-1], B.shape[1]), dtype=out_dtype) |
| 64 | |
| 65 | |
| 66 | if current_platform.is_rocm(): |
| 67 | direct_register_custom_op( |
| 68 | op_name="rocm_per_tensor_float_w8a8_scaled_mm_impl", |
| 69 | op_func=rocm_per_tensor_float_w8a8_scaled_mm_impl, |
| 70 | fake_impl=rocm_per_tensor_float_w8a8_scaled_mm_fake, |
| 71 | ) |
| 72 | |
| 73 | |
| 74 | class ROCmFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): |
| 75 | @classmethod |
| 76 | def is_supported( |
| 77 | cls, compute_capability: int | None = None |
| 78 | ) -> tuple[bool, str | None]: |
| 79 | if not current_platform.is_rocm(): |
| 80 | return False, "requires ROCm." |
| 81 | |
| 82 | from vllm.platforms.rocm import get_cdna_version, on_gfx12x, on_gfx1250 |
| 83 | |
| 84 | # wvSplitKQ (skinny GEMM) is excluded from the gfx1250 build. |
| 85 | if on_gfx1250(): |
| 86 | return False, "wvSplitKQ (skinny GEMM) is not built on gfx1250" |
| 87 | |
| 88 | # Restore RDNA4 (gfx12x) dropped by the get_cdna_version()>2 refactor. |
| 89 | if get_cdna_version() <= 2 and not on_gfx12x(): |
| 90 | return False, "requires CDNA3+ (gfx942/gfx950) or RDNA4 (gfx12x)" |
| 91 | |
| 92 | if not envs.VLLM_ROCM_USE_SKINNY_GEMM: |
| 93 | return False, "requires VLLM_ROCM_USE_SKINNY_GEMM to be enabled." |
| 94 | |
| 95 | return True, None |
| 96 | |
| 97 | @classmethod |
| 98 | def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]: |
| 99 | per_tensor_activation_scales = ( |
| 100 | c.activation_quant_key.scale.group_shape.is_per_tensor() |
| 101 | ) |
| 102 | per_tensor_weight_scales = c.weight_quant_key.scale.group_shape.is_per_tensor() |
| 103 | |
| 104 | if not (per_tensor_activation_scales and per_tensor_weight_scales): |
| 105 | return False, "requires per tensor activation and weight scales." |
| 106 | |
| 107 | return True, None |
| 108 | |
| 109 | def apply_scaled_mm( |
| 110 | self, |
| 111 | *, |
| 112 | A: torch.Tensor, |
| 113 | B: torch.Tensor, |
| 114 | out_dtype: torch.dtype, |
| 115 | As: torch.Tensor, |
| 116 | Bs: torch.Tensor, |
| 117 | bias: torch.Tensor | None, |
| 118 | output_shape: list, |
| 119 | ) -> torch.Tensor: |
| 120 | output = torch.ops.vllm.rocm_per_tensor_float_w8a8_scaled_mm_impl( |
| 121 | A, B, out_dtype, As, Bs, bias |
| 122 | ) |
| 123 | return torch.narrow(output, 0, 0, A.shape[0]).view(*output_shape) |