| 1 | #ifndef CPU_ATTN_NEON_HPP |
| 2 | #define CPU_ATTN_NEON_HPP |
| 3 | |
| 4 | #include "cpu_attn_impl.hpp" |
| 5 | #include <arm_neon.h> |
| 6 | #include <type_traits> |
| 7 | #ifdef ARM_BF16_SUPPORT |
| 8 | #include "cpu_attn_neon_bfmmla.hpp" |
| 9 | #endif |
| 10 | namespace cpu_attention { |
| 11 | |
| 12 | namespace { |
| 13 | |
| 14 | #define BLOCK_SIZE_ALIGNMENT 16 |
| 15 | #define HEAD_SIZE_ALIGNMENT 32 |
| 16 | #define MAX_Q_HEAD_NUM_PER_ITER 16 |
| 17 | |
| 18 | // These do not use vectorized class for loading / converting |
| 19 | // because csrc/cpu/cpu_types_arm.hpp does not have fallback options |
| 20 | // for vec_op::BF16Vec* / vec_op::BF16Vec* on Arm HW that |
| 21 | // doesn't support BF16. |
| 22 | // We don't use vec_op::FP32Vec* or vec_op::FP16Vec* for consistency. |
| 23 | template <typename kv_cache_t> |
| 24 | FORCE_INLINE void load_row8_B_as_f32(const kv_cache_t* p, float32x4_t& b0, |
| 25 | float32x4_t& b1); |
| 26 | |
| 27 | template <> |
| 28 | FORCE_INLINE void load_row8_B_as_f32<float>(const float* p, float32x4_t& b0, |
| 29 | float32x4_t& b1) { |
| 30 | b0 = vld1q_f32(p + 0); |
| 31 | b1 = vld1q_f32(p + 4); |
| 32 | } |
| 33 | |
| 34 | template <> |
| 35 | FORCE_INLINE void load_row8_B_as_f32<c10::Half>(const c10::Half* p, |
| 36 | float32x4_t& b0, |
| 37 | float32x4_t& b1) { |
| 38 | const float16_t* h = reinterpret_cast<const float16_t*>(p); |
| 39 | float16x8_t v = vld1q_f16(h); |
| 40 | b0 = vcvt_f32_f16(vget_low_f16(v)); |
| 41 | b1 = vcvt_f32_f16(vget_high_f16(v)); |
| 42 | } |
| 43 | |
| 44 | template <> |
| 45 | FORCE_INLINE void load_row8_B_as_f32<c10::BFloat16>(const c10::BFloat16* p, |
| 46 | float32x4_t& b0, |
| 47 | float32x4_t& b1) { |
| 48 | const uint16_t* u = reinterpret_cast<const uint16_t*>(p); |
| 49 | #ifdef ARM_BF16_SUPPORT |
| 50 | uint16x8_t u0 = vld1q_u16(u); |
| 51 | bfloat16x8_t bf0 = vreinterpretq_bf16_u16(u0); |
| 52 | b0 = vcvtq_low_f32_bf16(bf0); |
| 53 | b1 = vcvtq_high_f32_bf16(bf0); |
| 54 | #else |
| 55 | uint16x8_t x0 = vld1q_u16(u); |
| 56 | uint32x4_t lo = vshlq_n_u32(vmovl_u16(vget_low_u16(x0)), 16); |
| 57 | uint32x4_t hi = vshlq_n_u32(vmovl_u16(vget_high_u16(x0)), 16); |
| 58 | b0 = vreinterpretq_f32_u32(lo); |
| 59 | b1 = vreinterpretq_f32_u32(hi); |
| 60 | #endif |
| 61 | } |
| 62 | |
| 63 | // Mx8, with 1 <= M <= 8 , K streamed, unroll-by-4 with ASIMD FMLAs |
| 64 | // #Loads = (K // 4) * (M + 4 * sizeof(kv_cache_t) / 2) |
| 65 | // #FMLAs = (K // 4) * (4 * 2 * M) |
| 66 | // We have (4 * 2 * M) FMLAs for (M + 4 * sizeof(kv_cache_t) / 2) loads |
| 67 | template <int32_t M, typename kv_cache_t> |
| 68 | FORCE_INLINE void gemm_micro_neon_fmla_Mx8_Ku4( |
| 69 | const float* __restrict A, // [M x K], |
| 70 | const kv_cache_t* __restrict B, // [K x 8], |
| 71 | float* __restrict C, // [M x 8], |
| 72 | int64_t lda, int64_t ldb, int64_t ldc, int32_t K, bool accumulate) { |
| 73 | // kernel supports max M of 8, as it'd spill for larger M |
| 74 | static_assert(1 <= M && M <= 8, "M must be in [1,8]"); |
| 75 | |
| 76 | // helpers for per-M codegen |
| 77 | #define ROWS_APPLY(OP) OP(0) OP(1) OP(2) OP(3) OP(4) OP(5) OP(6) OP(7) |
| 78 | #define IF_M(i) if constexpr (M > (i)) |
| 79 | |
| 80 | // A row base pointers |
| 81 | #define DECL_A(i) const float* a##i = A + (i) * lda; |
| 82 | ROWS_APPLY(DECL_A) |
| 83 | #undef DECL_A |
| 84 | |
| 85 | // declare 2 accumulators per row of M |
| 86 | #define DECL_ACC(i) float32x4_t acc##i##_0, acc##i##_1; |
| 87 | ROWS_APPLY(DECL_ACC) |
| 88 | #undef DECL_ACC |
| 89 | |
| 90 | // initialize accumulators |
| 91 | #define INIT_ACC(i) \ |
| 92 | IF_M(i) { \ |
| 93 | if (accumulate) { \ |
| 94 | acc##i##_0 = vld1q_f32(C + (i) * ldc + 0); \ |
| 95 | acc##i##_1 = vld1q_f32(C + (i) * ldc + 4); \ |
| 96 | } else { \ |
| 97 | acc##i##_0 = vdupq_n_f32(0.f); \ |
| 98 | acc##i##_1 = vdupq_n_f32(0.f); \ |
| 99 | } \ |
| 100 | } |
| 101 | ROWS_APPLY(INIT_ACC) |
| 102 | #undef INIT_ACC |
| 103 | |
| 104 | int32_t k = 0; |
| 105 | |
| 106 | // K unrolled by 4 |
| 107 | for (; k + 3 < K; k += 4) { |
| 108 | // load A[k..k+3] for each active row (M) |
| 109 | #define LOAD_A4(i) \ |
| 110 | float32x4_t a##i##v; \ |
| 111 | IF_M(i) a##i##v = vld1q_f32(a##i + k); |
| 112 | ROWS_APPLY(LOAD_A4) |
| 113 | #undef LOAD_A4 |
| 114 | |
| 115 | // helper: FMA lane L from aiv |
| 116 | #define FMAS_LANE(i, aiv, L) \ |
| 117 | IF_M(i) { \ |
| 118 | acc##i##_0 = vfmaq_laneq_f32(acc##i##_0, b0, aiv, L); \ |
| 119 | acc##i##_1 = vfmaq_laneq_f32(acc##i##_1, b1, aiv, L); \ |
| 120 | } |
| 121 | |
| 122 | // k + 0 |
| 123 | { |
| 124 | float32x4_t b0, b1; |
| 125 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)(k + 0) * ldb, b0, b1); |
| 126 | #define STEP_K0(i) FMAS_LANE(i, a##i##v, 0) |
| 127 | ROWS_APPLY(STEP_K0) |
| 128 | #undef STEP_K0 |
| 129 | } |
| 130 | // k + 1 |
| 131 | { |
| 132 | float32x4_t b0, b1; |
| 133 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)(k + 1) * ldb, b0, b1); |
| 134 | #define STEP_K1(i) FMAS_LANE(i, a##i##v, 1) |
| 135 | ROWS_APPLY(STEP_K1) |
| 136 | #undef STEP_K1 |
| 137 | } |
| 138 | // k + 2 |
| 139 | { |
| 140 | float32x4_t b0, b1; |
| 141 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)(k + 2) * ldb, b0, b1); |
| 142 | #define STEP_K2(i) FMAS_LANE(i, a##i##v, 2) |
| 143 | ROWS_APPLY(STEP_K2) |
| 144 | #undef STEP_K2 |
| 145 | } |
| 146 | // k + 3 |
| 147 | { |
| 148 | float32x4_t b0, b1; |
| 149 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)(k + 3) * ldb, b0, b1); |
| 150 | #define STEP_K3(i) FMAS_LANE(i, a##i##v, 3) |
| 151 | ROWS_APPLY(STEP_K3) |
| 152 | #undef STEP_K3 |
| 153 | } |
| 154 | #undef FMAS_LANE |
| 155 | } |
| 156 | |
| 157 | // K tail |
| 158 | for (; k < K; ++k) { |
| 159 | float32x4_t b0, b1; |
| 160 | load_row8_B_as_f32<kv_cache_t>(B + (int64_t)k * ldb, b0, b1); |
| 161 | #define TAIL_ROW(i) \ |
| 162 | IF_M(i) { \ |
| 163 | float32x4_t ai = vdupq_n_f32(*(a##i + k)); \ |
| 164 | acc##i##_0 = vfmaq_f32(acc##i##_0, b0, ai); \ |
| 165 | acc##i##_1 = vfmaq_f32(acc##i##_1, b1, ai); \ |
| 166 | } |
| 167 | ROWS_APPLY(TAIL_ROW) |
| 168 | #undef TAIL_ROW |
| 169 | } |
| 170 | |
| 171 | // store accumulators to C |
| 172 | #define STORE_ROW(i) \ |
| 173 | IF_M(i) { \ |
| 174 | vst1q_f32(C + (i) * ldc + 0, acc##i##_0); \ |
| 175 | vst1q_f32(C + (i) * ldc + 4, acc##i##_1); \ |
| 176 | } |
| 177 | ROWS_APPLY(STORE_ROW) |
| 178 | #undef STORE_ROW |
| 179 | |
| 180 | #undef ROWS_APPLY |
| 181 | #undef IF_M |
| 182 | } |
| 183 | |
| 184 | template <int32_t N, typename kv_cache_t> |
| 185 | FORCE_INLINE void gemm_macro_neon_fmla_Mx8_Ku4(const float* __restrict A, |
| 186 | const kv_cache_t* __restrict B, |
| 187 | float* __restrict C, int32_t M, |
| 188 | int32_t K, int64_t lda, |
| 189 | int64_t ldb, int64_t ldc, |
| 190 | bool accumulate) { |
| 191 | // micro kernel is Mx8 |
| 192 | static_assert(N % 8 == 0, "N must be a multiple of 8"); |
| 193 | for (int32_t m = 0; m < M;) { |
| 194 | int32_t mb = (M - m >= 8) ? 8 : (M - m >= 4) ? 4 : (M - m >= 2) ? 2 : 1; |
| 195 | const float* Ab = A + m * lda; |
| 196 | float* Cb = C + m * ldc; |
| 197 | |
| 198 | for (int32_t n = 0; n < N; n += 8) { |
| 199 | const kv_cache_t* Bn = B + n; |
| 200 | float* Cn = Cb + n; |
| 201 | switch (mb) { |
| 202 | case 8: |
| 203 | gemm_micro_neon_fmla_Mx8_Ku4<8, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, |
| 204 | K, accumulate); |
| 205 | break; |
| 206 | case 4: |
| 207 | gemm_micro_neon_fmla_Mx8_Ku4<4, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, |
| 208 | K, accumulate); |
| 209 | break; |
| 210 | case 2: |
| 211 | gemm_micro_neon_fmla_Mx8_Ku4<2, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, |
| 212 | K, accumulate); |
| 213 | break; |
| 214 | default: |
| 215 | gemm_micro_neon_fmla_Mx8_Ku4<1, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, |
| 216 | K, accumulate); |
| 217 | break; |
| 218 | } |
| 219 | } |
| 220 | // no tail loop for N as it's guaranteed to be a multiple of 8 |
| 221 | m += mb; |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | template <typename kv_cache_t> |
| 226 | class TileGemmNeonFMLA { |
| 227 | public: |
| 228 | template <AttentionGemmPhase phase, int32_t k_size> |
| 229 | FORCE_INLINE static void gemm(const int32_t m_size, |
| 230 | float* __restrict__ a_tile, |
| 231 | kv_cache_t* __restrict__ b_tile, |
| 232 | float* __restrict__ c_tile, const int64_t lda, |
| 233 | const int64_t ldb, const int64_t ldc, |
| 234 | const int32_t block_size, |
| 235 | const int32_t dynamic_k_size, |
| 236 | const bool accum_c) { |
| 237 | if constexpr (phase == AttentionGemmPhase::QK) { |
| 238 | gemm_macro_neon_fmla_Mx8_Ku4<BLOCK_SIZE_ALIGNMENT, kv_cache_t>( |
| 239 | a_tile, b_tile, c_tile, m_size, k_size, lda, ldb, ldc, accum_c); |
| 240 | } else { |
| 241 | gemm_macro_neon_fmla_Mx8_Ku4<HEAD_SIZE_ALIGNMENT, kv_cache_t>( |
| 242 | a_tile, b_tile, c_tile, m_size, dynamic_k_size, lda, ldb, ldc, |
| 243 | accum_c); |
| 244 | } |
| 245 | } |
| 246 | }; |
| 247 | |
| 248 | } // namespace |
| 249 | |
| 250 | // this is similar to "ISA::VEC" at the moment |
| 251 | template <typename scalar_t, int64_t head_dim, typename kv_cache_scalar_t> |
| 252 | class AttentionImpl<ISA::NEON, scalar_t, head_dim, kv_cache_scalar_t> { |
| 253 | public: |
| 254 | using query_t = scalar_t; |
| 255 | using q_buffer_t = float; |
| 256 | using kv_cache_t = scalar_t; |
| 257 | using logits_buffer_t = float; |
| 258 | using partial_output_buffer_t = float; |
| 259 | using prob_buffer_t = float; |
| 260 | |
| 261 | constexpr static int64_t BlockSizeAlignment = |
| 262 | BLOCK_SIZE_ALIGNMENT; // KV token num unit of QK and PV phases |
| 263 | constexpr static int64_t HeadDimAlignment = |
| 264 | HEAD_SIZE_ALIGNMENT; // headdim num unit of PV phase |
| 265 | constexpr static int64_t MaxQHeadNumPerIteration = MAX_Q_HEAD_NUM_PER_ITER; |
| 266 | constexpr static int64_t HeadDim = head_dim; |
| 267 | constexpr static ISA ISAType = ISA::NEON; |
| 268 | constexpr static bool scale_on_logits = false; // apply scale on q_buffer |
| 269 | |
| 270 | static_assert(HeadDim % HeadDimAlignment == 0); |
| 271 | // the gemm micro kernel is Mx8 |
| 272 | static_assert(HeadDimAlignment % 8 == 0); |
| 273 | static_assert(BlockSizeAlignment % 8 == 0); |
| 274 | |
| 275 | public: |
| 276 | template <template <typename tile_gemm_t> typename attention> |
| 277 | FORCE_INLINE void execute_attention(DEFINE_CPU_ATTENTION_PARAMS) { |
| 278 | attention<TileGemmNeonFMLA<kv_cache_t>> attention_iteration; |
| 279 | attention_iteration(CPU_ATTENTION_PARAMS); |
| 280 | } |
| 281 | |
| 282 | // k_cache_token_group_stride: stride of K cache when move to next |
| 283 | // BlockSizeAlignment tokens in a block |
| 284 | constexpr static int64_t k_cache_token_group_stride( |
| 285 | const int32_t block_size) { |
| 286 | return BlockSizeAlignment; // layout of k_cache block is [head_dim, |
| 287 | // block_size], row-major |
| 288 | } |
| 289 | |
| 290 | // v_cache_token_group_stride: stride of V cache when move to next |
| 291 | // BlockSizeAlignment tokens in a block |
| 292 | constexpr static int64_t v_cache_token_group_stride( |
| 293 | const int32_t block_size) { |
| 294 | return head_dim * BlockSizeAlignment; // layout of v_cache is [block_size, |
| 295 | // head_dim], row-major |
| 296 | } |
| 297 | |
| 298 | // v_cache_head_group_stride: stride of V cache when move to next |
| 299 | // HeadDimAlignment head dims in a block |
| 300 | constexpr static int64_t v_cache_head_group_stride(const int32_t block_size) { |
| 301 | return HeadDimAlignment; // layout of v_cache is [block_size, head_dim], |
| 302 | // row-major |
| 303 | } |
| 304 | |
| 305 | // Copy q to q_buffer and cast it to fp32 |
| 306 | static void copy_q_heads_tile( |
| 307 | scalar_t* __restrict__ src, // [q_num, q_heads_per_kv, head_size] |
| 308 | float* __restrict__ q_buffer, const int32_t q_num, |
| 309 | const int32_t q_heads_per_kv, const int64_t q_num_stride, |
| 310 | const int64_t q_head_stride, float scale) { |
| 311 | static_assert(head_dim % 16 == 0); |
| 312 | constexpr int32_t unroll_size = head_dim / 16; |
| 313 | using load_vec_t = typename VecTypeTrait<scalar_t>::vec_t; |
| 314 | |
| 315 | vec_op::FP32Vec16 scale_vec(scale); |
| 316 | for (int32_t q_num_idx = 0; q_num_idx < q_num; ++q_num_idx) { |
| 317 | for (int32_t q_head_idx = 0; q_head_idx < q_heads_per_kv; ++q_head_idx) { |
| 318 | scalar_t* __restrict__ curr_q = |
| 319 | src + q_num_idx * q_num_stride + q_head_idx * q_head_stride; |
| 320 | float* __restrict__ curr_q_buffer = |
| 321 | q_buffer + q_num_idx * q_heads_per_kv * head_dim + |
| 322 | q_head_idx * head_dim; |
| 323 | |
| 324 | vec_op::unroll_loop<int32_t, unroll_size>([&](int32_t i) { |
| 325 | load_vec_t vec(curr_q); |
| 326 | vec_op::FP32Vec16 fp32_vec(vec); |
| 327 | fp32_vec = fp32_vec * scale_vec; |
| 328 | fp32_vec.save(curr_q_buffer); |
| 329 | |
| 330 | curr_q += 16; |
| 331 | curr_q_buffer += 16; |
| 332 | }); |
| 333 | } |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | // reshape K as column-major and V as row-major |
| 338 | static void reshape_and_cache( |
| 339 | const scalar_t* __restrict__ key, const scalar_t* __restrict__ value, |
| 340 | scalar_t* __restrict__ key_cache, scalar_t* __restrict__ value_cache, |
| 341 | const int64_t* __restrict__ slot_mapping, const int64_t token_num, |
| 342 | const int64_t key_token_num_stride, const int64_t value_token_num_stride, |
| 343 | const int64_t head_num, const int64_t key_head_num_stride, |
| 344 | const int64_t value_head_num_stride, const int64_t num_blocks, |
| 345 | const int64_t num_blocks_stride, const int64_t cache_head_num_stride, |
| 346 | const int64_t block_size, const int64_t block_size_stride, |
| 347 | const float /*k_inv*/ = 0.0f, const float /*v_inv*/ = 0.0f) { |
| 348 | #pragma omp parallel for collapse(2) |
| 349 | for (int64_t token_idx = 0; token_idx < token_num; ++token_idx) { |
| 350 | for (int64_t head_idx = 0; head_idx < head_num; ++head_idx) { |
| 351 | const int64_t pos = slot_mapping[token_idx]; |
| 352 | if (pos < 0) { |
| 353 | // skip |
| 354 | continue; |
| 355 | } |
| 356 | |
| 357 | const int64_t block_idx = pos / block_size; |
| 358 | const int64_t block_offset = pos % block_size; |
| 359 | { |
| 360 | // Write Key |
| 361 | const scalar_t* key_start_ptr = key + |
| 362 | token_idx * key_token_num_stride + |
| 363 | head_idx * key_head_num_stride; |
| 364 | scalar_t* key_cache_start_ptr = |
| 365 | key_cache + block_idx * num_blocks_stride + |
| 366 | head_idx * cache_head_num_stride + block_offset; |
| 367 | |
| 368 | #pragma GCC unroll 8 |
| 369 | for (int64_t i = 0, j = 0; i < head_dim; ++i, j += block_size) { |
| 370 | key_cache_start_ptr[j] = key_start_ptr[i]; |
| 371 | } |
| 372 | } |
| 373 | { |
| 374 | // Write Value |
| 375 | const scalar_t* value_start_ptr = value + |
| 376 | token_idx * value_token_num_stride + |
| 377 | head_idx * value_head_num_stride; |
| 378 | scalar_t* value_cache_start_ptr = |
| 379 | value_cache + block_idx * num_blocks_stride + |
| 380 | head_idx * cache_head_num_stride + block_offset * head_dim; |
| 381 | std::memcpy(value_cache_start_ptr, value_start_ptr, |
| 382 | sizeof(scalar_t) * head_dim); |
| 383 | } |
| 384 | } |
| 385 | } |
| 386 | } |
| 387 | }; |
| 388 | |
| 389 | #ifdef ARM_BF16_SUPPORT |
| 390 | // For BF16 on Arm, reuse the BFMMLA kernels with the same token alignment. |
| 391 | template <int64_t head_dim> |
| 392 | class AttentionImpl<ISA::NEON, c10::BFloat16, head_dim, c10::BFloat16> |
| 393 | : public AttentionImplNEONBFMMLA<BLOCK_SIZE_ALIGNMENT, ISA::NEON, |
| 394 | head_dim> {}; |
| 395 | #endif |
| 396 | } // namespace cpu_attention |
| 397 | |
| 398 | #undef BLOCK_SIZE_ALIGNMENT |
| 399 | #undef HEAD_SIZE_ALIGNMENT |
| 400 | #undef MAX_Q_HEAD_NUM_PER_ITER |
| 401 | |
| 402 | #endif // #ifndef CPU_ATTN_ASIMD_HPP |