| 1 | // SPDX-License-Identifier: Apache-2.0 |
| 2 | // SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | #ifndef CPU_ATTN_RVV_HPP |
| 5 | #define CPU_ATTN_RVV_HPP |
| 6 | |
| 7 | // RVV attention kernel using VLEN-agnostic RVVI() macros from |
| 8 | // cpu_types_riscv_defs.hpp. The Mx8 tile GEMM uses 8 FP32 elements |
| 9 | // per vector (LMUL_256 bits of FP32 data), which maps to: |
| 10 | // VLEN=128: m2 (256 bits = 8 x FP32) |
| 11 | // VLEN=256: m1 (256 bits = 8 x FP32) |
| 12 | // Only VLEN=128 and VLEN=256 are supported; other VLENs (512, 1024) |
| 13 | // and scalar RISC-V builds fall back to VEC/VEC16. |
| 14 | #if defined(__riscv_v_min_vlen) && \ |
| 15 | (__riscv_v_min_vlen == 128 || __riscv_v_min_vlen == 256) |
| 16 | |
| 17 | #include "cpu_attn_impl.hpp" |
| 18 | #include "cpu_types_riscv_defs.hpp" |
| 19 | #include <riscv_vector.h> |
| 20 | #include <type_traits> |
| 21 | |
| 22 | namespace cpu_attention { |
| 23 | |
| 24 | namespace { |
| 25 | |
| 26 | #define BLOCK_SIZE_ALIGNMENT 32 |
| 27 | #define HEAD_SIZE_ALIGNMENT 32 |
| 28 | #define MAX_Q_HEAD_NUM_PER_ITER 16 |
| 29 | |
| 30 | // ============================================================================ |
| 31 | // B-matrix row loading: load 8 elements as FP32 |
| 32 | // ============================================================================ |
| 33 | |
| 34 | template <typename kv_cache_t> |
| 35 | FORCE_INLINE fixed_fp32x8_t load_row8_B_as_f32(const kv_cache_t* p); |
| 36 | |
| 37 | template <> |
| 38 | FORCE_INLINE fixed_fp32x8_t load_row8_B_as_f32<float>(const float* p) { |
| 39 | return RVVI(__riscv_vle32_v_f32, LMUL_256)(p, 8); |
| 40 | } |
| 41 | |
| 42 | template <> |
| 43 | FORCE_INLINE fixed_fp32x8_t load_row8_B_as_f32<c10::Half>(const c10::Half* p) { |
| 44 | #ifdef __riscv_zvfh |
| 45 | fixed_fp16x8_t h = RVVI(__riscv_vle16_v_f16, LMUL_128)( |
| 46 | reinterpret_cast<const _Float16*>(p), 8); |
| 47 | return RVVI(__riscv_vfwcvt_f_f_v_f32, LMUL_256)(h, 8); |
| 48 | #else |
| 49 | alignas(16) float tmp[8]; |
| 50 | for (int i = 0; i < 8; ++i) { |
| 51 | tmp[i] = static_cast<float>(p[i]); |
| 52 | } |
| 53 | return RVVI(__riscv_vle32_v_f32, LMUL_256)(tmp, 8); |
| 54 | #endif |
| 55 | } |
| 56 | |
| 57 | template <> |
| 58 | FORCE_INLINE fixed_fp32x8_t |
| 59 | load_row8_B_as_f32<c10::BFloat16>(const c10::BFloat16* p) { |
| 60 | #ifdef __riscv_zvfbfmin |
| 61 | fixed_bf16x8_t bf = RVVI(__riscv_vle16_v_bf16, LMUL_128)( |
| 62 | reinterpret_cast<const __bf16*>(p), 8); |
| 63 | return RVVI(__riscv_vfwcvtbf16_f_f_v_f32, LMUL_256)(bf, 8); |
| 64 | #else |
| 65 | fixed_u16x8_t raw = RVVI(__riscv_vle16_v_u16, LMUL_128)( |
| 66 | reinterpret_cast<const uint16_t*>(p), 8); |
| 67 | fixed_u32x8_t wide = RVVI(__riscv_vzext_vf2_u32, LMUL_256)(raw, 8); |
| 68 | fixed_u32x8_t shifted = RVVI(__riscv_vsll_vx_u32, LMUL_256)(wide, 16, 8); |
| 69 | return RVVI4(__riscv_vreinterpret_v_u32, LMUL_256, _f32, LMUL_256)(shifted); |
| 70 | #endif |
| 71 | } |
| 72 | |
| 73 | // ============================================================================ |
| 74 | // Micro kernel: Mx8 tile, K unrolled by 4, RVV scalar-broadcast FMA |
| 75 | // ============================================================================ |
| 76 | // |
| 77 | // RVV has no lane-indexed FMA; instead we load A elements as scalars and |
| 78 | // use vfmacc_vf (scalar * vector + accumulator). |
| 79 | // |
| 80 | // The 8-column tile uses LMUL_256 bits of FP32 data: |
| 81 | // VLEN=128: m2 (2 regs per accumulator), M=8 => 18 of 32 regs |
| 82 | // VLEN=256: m1 (1 reg per accumulator), M=8 => 9 of 32 regs |
| 83 | |
| 84 | template <int32_t M, typename kv_cache_t> |
| 85 | FORCE_INLINE void gemm_micro_rvv_fma_Mx8_Ku4( |
| 86 | const float* __restrict A, // [M x K] |
| 87 | const kv_cache_t* __restrict B, // [K x 8] |
| 88 | float* __restrict C, // [M x 8] |
| 89 | int64_t lda, int64_t ldb, int64_t ldc, int32_t K, bool accumulate) { |
| 90 | static_assert(1 <= M && M <= 8, "M must be in [1,8]"); |
| 91 | |
| 92 | constexpr size_t vl = 8; |
| 93 | |
| 94 | #define ROWS_APPLY(OP) OP(0) OP(1) OP(2) OP(3) OP(4) OP(5) OP(6) OP(7) |
| 95 | #define IF_M(i) if constexpr (M > (i)) |
| 96 | |
| 97 | #define DECL_A(i) const float* a##i = A + (i) * lda; |
| 98 | ROWS_APPLY(DECL_A) |
| 99 | #undef DECL_A |
| 100 | |
| 101 | #define DECL_ACC(i) fixed_fp32x8_t acc##i; |
| 102 | ROWS_APPLY(DECL_ACC) |
| 103 | #undef DECL_ACC |
| 104 | |
| 105 | #define INIT_ACC(i) \ |
| 106 | IF_M(i) { \ |
| 107 | if (accumulate) { \ |
| 108 | acc##i = RVVI(__riscv_vle32_v_f32, LMUL_256)(C + (i) * ldc, vl); \ |
| 109 | } else { \ |
| 110 | acc##i = RVVI(__riscv_vfmv_v_f_f32, LMUL_256)(0.f, vl); \ |
| 111 | } \ |
| 112 | } |
| 113 | ROWS_APPLY(INIT_ACC) |
| 114 | #undef INIT_ACC |
| 115 | |
| 116 | int32_t k = 0; |
| 117 | |
| 118 | for (; k + 3 < K; k += 4) { |
| 119 | { |
| 120 | fixed_fp32x8_t b = |
| 121 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)(k + 0) * ldb); |
| 122 | #define STEP_K0(i) \ |
| 123 | IF_M(i) { \ |
| 124 | acc##i = RVVI(__riscv_vfmacc_vf_f32, LMUL_256)(acc##i, *(a##i + k + 0), \ |
| 125 | b, vl); \ |
| 126 | } |
| 127 | ROWS_APPLY(STEP_K0) |
| 128 | #undef STEP_K0 |
| 129 | } |
| 130 | { |
| 131 | fixed_fp32x8_t b = |
| 132 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)(k + 1) * ldb); |
| 133 | #define STEP_K1(i) \ |
| 134 | IF_M(i) { \ |
| 135 | acc##i = RVVI(__riscv_vfmacc_vf_f32, LMUL_256)(acc##i, *(a##i + k + 1), \ |
| 136 | b, vl); \ |
| 137 | } |
| 138 | ROWS_APPLY(STEP_K1) |
| 139 | #undef STEP_K1 |
| 140 | } |
| 141 | { |
| 142 | fixed_fp32x8_t b = |
| 143 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)(k + 2) * ldb); |
| 144 | #define STEP_K2(i) \ |
| 145 | IF_M(i) { \ |
| 146 | acc##i = RVVI(__riscv_vfmacc_vf_f32, LMUL_256)(acc##i, *(a##i + k + 2), \ |
| 147 | b, vl); \ |
| 148 | } |
| 149 | ROWS_APPLY(STEP_K2) |
| 150 | #undef STEP_K2 |
| 151 | } |
| 152 | { |
| 153 | fixed_fp32x8_t b = |
| 154 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)(k + 3) * ldb); |
| 155 | #define STEP_K3(i) \ |
| 156 | IF_M(i) { \ |
| 157 | acc##i = RVVI(__riscv_vfmacc_vf_f32, LMUL_256)(acc##i, *(a##i + k + 3), \ |
| 158 | b, vl); \ |
| 159 | } |
| 160 | ROWS_APPLY(STEP_K3) |
| 161 | #undef STEP_K3 |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | for (; k < K; ++k) { |
| 166 | fixed_fp32x8_t b = load_row8_B_as_f32<kv_cache_t>(B + (int64_t)k * ldb); |
| 167 | #define TAIL_ROW(i) \ |
| 168 | IF_M(i) { \ |
| 169 | acc##i = \ |
| 170 | RVVI(__riscv_vfmacc_vf_f32, LMUL_256)(acc##i, *(a##i + k), b, vl); \ |
| 171 | } |
| 172 | ROWS_APPLY(TAIL_ROW) |
| 173 | #undef TAIL_ROW |
| 174 | } |
| 175 | |
| 176 | #define STORE_ROW(i) \ |
| 177 | IF_M(i) { RVVI(__riscv_vse32_v_f32, LMUL_256)(C + (i) * ldc, acc##i, vl); } |
| 178 | ROWS_APPLY(STORE_ROW) |
| 179 | #undef STORE_ROW |
| 180 | |
| 181 | #undef ROWS_APPLY |
| 182 | #undef IF_M |
| 183 | } |
| 184 | |
| 185 | // ============================================================================ |
| 186 | // Macro kernel: dispatch M tiles of {8,4,2,1}, step N by 8 |
| 187 | // ============================================================================ |
| 188 | |
| 189 | template <int32_t N, typename kv_cache_t> |
| 190 | FORCE_INLINE void gemm_macro_rvv_fma_Mx8_Ku4(const float* __restrict A, |
| 191 | const kv_cache_t* __restrict B, |
| 192 | float* __restrict C, int32_t M, |
| 193 | int32_t K, int64_t lda, |
| 194 | int64_t ldb, int64_t ldc, |
| 195 | bool accumulate) { |
| 196 | static_assert(N % 8 == 0, "N must be a multiple of 8"); |
| 197 | for (int32_t m = 0; m < M;) { |
| 198 | int32_t mb = (M - m >= 8) ? 8 : (M - m >= 4) ? 4 : (M - m >= 2) ? 2 : 1; |
| 199 | const float* Ab = A + m * lda; |
| 200 | float* Cb = C + m * ldc; |
| 201 | |
| 202 | for (int32_t n = 0; n < N; n += 8) { |
| 203 | const kv_cache_t* Bn = B + n; |
| 204 | float* Cn = Cb + n; |
| 205 | switch (mb) { |
| 206 | case 8: |
| 207 | gemm_micro_rvv_fma_Mx8_Ku4<8, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, |
| 208 | K, accumulate); |
| 209 | break; |
| 210 | case 4: |
| 211 | gemm_micro_rvv_fma_Mx8_Ku4<4, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, |
| 212 | K, accumulate); |
| 213 | break; |
| 214 | case 2: |
| 215 | gemm_micro_rvv_fma_Mx8_Ku4<2, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, |
| 216 | K, accumulate); |
| 217 | break; |
| 218 | default: |
| 219 | gemm_micro_rvv_fma_Mx8_Ku4<1, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, |
| 220 | K, accumulate); |
| 221 | break; |
| 222 | } |
| 223 | } |
| 224 | m += mb; |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | // ============================================================================ |
| 229 | // TileGemm wrapper — plugs into AttentionMainLoop |
| 230 | // ============================================================================ |
| 231 | |
| 232 | template <typename kv_cache_t> |
| 233 | class TileGemmRVV { |
| 234 | public: |
| 235 | template <AttentionGemmPhase phase, int32_t k_size> |
| 236 | FORCE_INLINE static void gemm(const int32_t m_size, |
| 237 | float* __restrict__ a_tile, |
| 238 | kv_cache_t* __restrict__ b_tile, |
| 239 | float* __restrict__ c_tile, const int64_t lda, |
| 240 | const int64_t ldb, const int64_t ldc, |
| 241 | const int32_t block_size, |
| 242 | const int32_t dynamic_k_size, |
| 243 | const bool accum_c) { |
| 244 | if constexpr (phase == AttentionGemmPhase::QK) { |
| 245 | gemm_macro_rvv_fma_Mx8_Ku4<BLOCK_SIZE_ALIGNMENT, kv_cache_t>( |
| 246 | a_tile, b_tile, c_tile, m_size, k_size, lda, ldb, ldc, accum_c); |
| 247 | } else { |
| 248 | gemm_macro_rvv_fma_Mx8_Ku4<HEAD_SIZE_ALIGNMENT, kv_cache_t>( |
| 249 | a_tile, b_tile, c_tile, m_size, dynamic_k_size, lda, ldb, ldc, |
| 250 | accum_c); |
| 251 | } |
| 252 | } |
| 253 | }; |
| 254 | |
| 255 | } // namespace |
| 256 | |
| 257 | // ============================================================================ |
| 258 | // AttentionImpl<ISA::RVV> — mirrors ISA::NEON specialization |
| 259 | // ============================================================================ |
| 260 | |
| 261 | template <typename scalar_t, int64_t head_dim, typename kv_cache_scalar_t> |
| 262 | class AttentionImpl<ISA::RVV, scalar_t, head_dim, kv_cache_scalar_t> { |
| 263 | public: |
| 264 | using query_t = scalar_t; |
| 265 | using q_buffer_t = float; |
| 266 | using kv_cache_t = scalar_t; |
| 267 | using logits_buffer_t = float; |
| 268 | using partial_output_buffer_t = float; |
| 269 | using prob_buffer_t = float; |
| 270 | |
| 271 | constexpr static int64_t BlockSizeAlignment = BLOCK_SIZE_ALIGNMENT; |
| 272 | constexpr static int64_t HeadDimAlignment = HEAD_SIZE_ALIGNMENT; |
| 273 | constexpr static int64_t MaxQHeadNumPerIteration = MAX_Q_HEAD_NUM_PER_ITER; |
| 274 | constexpr static int64_t HeadDim = head_dim; |
| 275 | constexpr static ISA ISAType = ISA::RVV; |
| 276 | constexpr static bool scale_on_logits = false; |
| 277 | |
| 278 | static_assert(HeadDim % HeadDimAlignment == 0); |
| 279 | static_assert(HeadDimAlignment % 8 == 0); |
| 280 | static_assert(BlockSizeAlignment % 8 == 0); |
| 281 | |
| 282 | public: |
| 283 | template <template <typename tile_gemm_t> typename attention> |
| 284 | FORCE_INLINE void execute_attention(DEFINE_CPU_ATTENTION_PARAMS) { |
| 285 | attention<TileGemmRVV<kv_cache_t>> attention_iteration; |
| 286 | attention_iteration(CPU_ATTENTION_PARAMS); |
| 287 | } |
| 288 | |
| 289 | constexpr static int64_t k_cache_token_group_stride( |
| 290 | const int32_t block_size) { |
| 291 | return BlockSizeAlignment; |
| 292 | } |
| 293 | |
| 294 | constexpr static int64_t v_cache_token_group_stride( |
| 295 | const int32_t block_size) { |
| 296 | return head_dim * BlockSizeAlignment; |
| 297 | } |
| 298 | |
| 299 | constexpr static int64_t v_cache_head_group_stride(const int32_t block_size) { |
| 300 | return HeadDimAlignment; |
| 301 | } |
| 302 | |
| 303 | static void copy_q_heads_tile(scalar_t* __restrict__ src, |
| 304 | float* __restrict__ q_buffer, |
| 305 | const int32_t q_num, |
| 306 | const int32_t q_heads_per_kv, |
| 307 | const int64_t q_num_stride, |
| 308 | const int64_t q_head_stride, float scale) { |
| 309 | static_assert(head_dim % 16 == 0); |
| 310 | constexpr int32_t unroll_size = head_dim / 16; |
| 311 | using load_vec_t = typename VecTypeTrait<scalar_t>::vec_t; |
| 312 | |
| 313 | vec_op::FP32Vec16 scale_vec(scale); |
| 314 | for (int32_t q_num_idx = 0; q_num_idx < q_num; ++q_num_idx) { |
| 315 | for (int32_t q_head_idx = 0; q_head_idx < q_heads_per_kv; ++q_head_idx) { |
| 316 | scalar_t* __restrict__ curr_q = |
| 317 | src + q_num_idx * q_num_stride + q_head_idx * q_head_stride; |
| 318 | float* __restrict__ curr_q_buffer = |
| 319 | q_buffer + q_num_idx * q_heads_per_kv * head_dim + |
| 320 | q_head_idx * head_dim; |
| 321 | |
| 322 | vec_op::unroll_loop<int32_t, unroll_size>([&](int32_t i) { |
| 323 | load_vec_t vec(curr_q); |
| 324 | vec_op::FP32Vec16 fp32_vec(vec); |
| 325 | fp32_vec = fp32_vec * scale_vec; |
| 326 | fp32_vec.save(curr_q_buffer); |
| 327 | |
| 328 | curr_q += 16; |
| 329 | curr_q_buffer += 16; |
| 330 | }); |
| 331 | } |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | static void reshape_and_cache( |
| 336 | const scalar_t* __restrict__ key, const scalar_t* __restrict__ value, |
| 337 | scalar_t* __restrict__ key_cache, scalar_t* __restrict__ value_cache, |
| 338 | const int64_t* __restrict__ slot_mapping, const int64_t token_num, |
| 339 | const int64_t key_token_num_stride, const int64_t value_token_num_stride, |
| 340 | const int64_t head_num, const int64_t key_head_num_stride, |
| 341 | const int64_t value_head_num_stride, const int64_t num_blocks, |
| 342 | const int64_t num_blocks_stride, const int64_t cache_head_num_stride, |
| 343 | const int64_t block_size, const int64_t block_size_stride, |
| 344 | const float /*k_inv*/ = 0.0f, const float /*v_inv*/ = 0.0f) { |
| 345 | #pragma omp parallel for collapse(2) |
| 346 | for (int64_t token_idx = 0; token_idx < token_num; ++token_idx) { |
| 347 | for (int64_t head_idx = 0; head_idx < head_num; ++head_idx) { |
| 348 | const int64_t pos = slot_mapping[token_idx]; |
| 349 | if (pos < 0) { |
| 350 | continue; |
| 351 | } |
| 352 | |
| 353 | const int64_t block_idx = pos / block_size; |
| 354 | const int64_t block_offset = pos % block_size; |
| 355 | { |
| 356 | const scalar_t* key_start_ptr = key + |
| 357 | token_idx * key_token_num_stride + |
| 358 | head_idx * key_head_num_stride; |
| 359 | scalar_t* key_cache_start_ptr = |
| 360 | key_cache + block_idx * num_blocks_stride + |
| 361 | head_idx * cache_head_num_stride + block_offset; |
| 362 | |
| 363 | { |
| 364 | const ptrdiff_t byte_stride = block_size * sizeof(scalar_t); |
| 365 | int64_t i = 0; |
| 366 | for (; i < head_dim;) { |
| 367 | size_t vl; |
| 368 | if constexpr (std::is_same_v<scalar_t, float>) { |
| 369 | vl = __riscv_vsetvl_e32m2(head_dim - i); |
| 370 | vfloat32m2_t v = __riscv_vle32_v_f32m2( |
| 371 | reinterpret_cast<const float*>(key_start_ptr + i), vl); |
| 372 | __riscv_vsse32_v_f32m2( |
| 373 | reinterpret_cast<float*>(key_cache_start_ptr + |
| 374 | i * block_size), |
| 375 | byte_stride, v, vl); |
| 376 | } else { |
| 377 | vl = __riscv_vsetvl_e16m1(head_dim - i); |
| 378 | vuint16m1_t v = __riscv_vle16_v_u16m1( |
| 379 | reinterpret_cast<const uint16_t*>(key_start_ptr + i), vl); |
| 380 | __riscv_vsse16_v_u16m1( |
| 381 | reinterpret_cast<uint16_t*>(key_cache_start_ptr + |
| 382 | i * block_size), |
| 383 | byte_stride, v, vl); |
| 384 | } |
| 385 | i += vl; |
| 386 | } |
| 387 | } |
| 388 | } |
| 389 | { |
| 390 | const scalar_t* value_start_ptr = value + |
| 391 | token_idx * value_token_num_stride + |
| 392 | head_idx * value_head_num_stride; |
| 393 | scalar_t* value_cache_start_ptr = |
| 394 | value_cache + block_idx * num_blocks_stride + |
| 395 | head_idx * cache_head_num_stride + block_offset * head_dim; |
| 396 | std::memcpy(value_cache_start_ptr, value_start_ptr, |
| 397 | sizeof(scalar_t) * head_dim); |
| 398 | } |
| 399 | } |
| 400 | } |
| 401 | } |
| 402 | }; |
| 403 | |
| 404 | } // namespace cpu_attention |
| 405 | |
| 406 | #undef BLOCK_SIZE_ALIGNMENT |
| 407 | #undef HEAD_SIZE_ALIGNMENT |
| 408 | #undef MAX_Q_HEAD_NUM_PER_ITER |
| 409 | |
| 410 | #endif // __riscv_v_min_vlen == 128 || 256 |
| 411 | |
| 412 | #endif // CPU_ATTN_RVV_HPP |