| 1 | #include "cpu/cpu_types.hpp" |
| 2 | #include "cpu/utils.hpp" |
| 3 | |
| 4 | #ifdef CPU_CAPABILITY_AMXBF16 |
| 5 | #include "cpu/micro_gemm/cpu_micro_gemm_amx.hpp" |
| 6 | #endif |
| 7 | #if defined(__riscv_v) |
| 8 | #include "cpu/micro_gemm/cpu_micro_gemm_rvv.hpp" |
| 9 | #endif |
| 10 | #if defined(__powerpc__) |
| 11 | #include "cpu/micro_gemm/cpu_micro_gemm_vsx.hpp" |
| 12 | #endif |
| 13 | #include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp" |
| 14 | |
| 15 | #define VLLM_DISPATCH_CASE_16B_TYPES(...) \ |
| 16 | AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \ |
| 17 | AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) |
| 18 | |
| 19 | #define VLLM_DISPATCH_16B_TYPES(TYPE, NAME, ...) \ |
| 20 | AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_16B_TYPES(__VA_ARGS__)) |
| 21 | |
| 22 | template <typename T> |
| 23 | void print_logits(const char* name, T* ptr, int32_t row, int32_t col, |
| 24 | int32_t stride) { |
| 25 | std::stringstream ss; |
| 26 | ss << std::fixed << std::setprecision(5) << name << ": [\n"; |
| 27 | auto* curr_logits_buffer = ptr; |
| 28 | for (int32_t m = 0; m < row; ++m) { |
| 29 | for (int32_t n = 0; n < col; ++n) { |
| 30 | ss << curr_logits_buffer[n] << ", "; |
| 31 | } |
| 32 | ss << "\n"; |
| 33 | curr_logits_buffer += stride; |
| 34 | } |
| 35 | ss << "]\n"; |
| 36 | std::printf("%s", ss.str().c_str()); |
| 37 | } |
| 38 | |
| 39 | namespace { |
| 40 | using cpu_utils::ISA; |
| 41 | using cpu_utils::VecTypeTrait; |
| 42 | |
| 43 | template <typename scalar_t, ISA isa, bool has_zp> |
| 44 | class Dequantizer4b { |
| 45 | public: |
| 46 | constexpr static int32_t pack_num = 32 / 4; |
| 47 | using scalar_vec_t = typename VecTypeTrait<scalar_t>::vec_t; |
| 48 | |
| 49 | public: |
| 50 | static void dequant(int32_t* __restrict__ q_weight, |
| 51 | scalar_t* __restrict__ weight, |
| 52 | scalar_t* __restrict__ scales, |
| 53 | int32_t* __restrict__ zeros, const int64_t scales_stride, |
| 54 | const int64_t zeros_stride, const int32_t k_size, |
| 55 | const int32_t group_size) { |
| 56 | vec_op::FP32Vec16 lut; |
| 57 | if constexpr (has_zp) { |
| 58 | // AWQ |
| 59 | alignas(64) static const float LUT[16] = { |
| 60 | 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, |
| 61 | 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}; |
| 62 | lut = vec_op::FP32Vec16(LUT); |
| 63 | } else { |
| 64 | // GPTQ |
| 65 | alignas(64) static const float LUT[16] = { |
| 66 | -8.0f, -7.0f, -6.0f, -5.0f, -4.0f, -3.0f, -2.0f, -1.0f, |
| 67 | 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}; |
| 68 | lut = vec_op::FP32Vec16(LUT); |
| 69 | } |
| 70 | |
| 71 | // per 64-bits elem contains 16 output channels |
| 72 | int64_t* __restrict__ curr_q_weight = reinterpret_cast<int64_t*>(q_weight); |
| 73 | int64_t* __restrict__ curr_zeros = reinterpret_cast<int64_t*>(zeros); |
| 74 | scalar_t* __restrict__ curr_weight = weight; |
| 75 | scalar_t* __restrict__ curr_scale = scales; |
| 76 | vec_op::FP32Vec16 scale_0; |
| 77 | vec_op::FP32Vec16 scale_1; |
| 78 | vec_op::FP32Vec16 zero_0; |
| 79 | vec_op::FP32Vec16 zero_1; |
| 80 | int32_t group_counter = 0; |
| 81 | for (int32_t k_idx = 0; k_idx < k_size; k_idx += 2) { |
| 82 | int64_t qwb_0 = *curr_q_weight; |
| 83 | int64_t qwb_1 = *(curr_q_weight + 1); |
| 84 | vec_op::FP32Vec16 wb_0(qwb_0, lut); |
| 85 | vec_op::FP32Vec16 wb_1(qwb_1, lut); |
| 86 | |
| 87 | if (group_counter == 0) { |
| 88 | scale_0 = vec_op::FP32Vec16(scalar_vec_t(curr_scale)); |
| 89 | scale_1 = vec_op::FP32Vec16(scale_0); |
| 90 | curr_scale += scales_stride; |
| 91 | |
| 92 | if constexpr (has_zp) { |
| 93 | zero_0 = vec_op::FP32Vec16(*curr_zeros, lut); |
| 94 | zero_1 = vec_op::FP32Vec16(zero_0); |
| 95 | curr_zeros += zeros_stride / 2; |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | if constexpr (has_zp) { |
| 100 | wb_0 = wb_0 - zero_0; |
| 101 | wb_1 = wb_1 - zero_1; |
| 102 | } |
| 103 | |
| 104 | wb_0 = wb_0 * scale_0; |
| 105 | wb_1 = wb_1 * scale_1; |
| 106 | |
| 107 | scalar_vec_t output_vec_0(wb_0); |
| 108 | scalar_vec_t output_vec_1(wb_1); |
| 109 | |
| 110 | // AMX and VSX need to interleave K elements to pack as 32 bits |
| 111 | if constexpr (isa == ISA::AMX || isa == ISA::VSX) { |
| 112 | vec_op::interleave_save(output_vec_0, output_vec_1, curr_weight); |
| 113 | } else { |
| 114 | output_vec_0.save(curr_weight); |
| 115 | output_vec_1.save(curr_weight + 16); |
| 116 | } |
| 117 | |
| 118 | // update |
| 119 | curr_q_weight += 2; |
| 120 | curr_weight += 32; |
| 121 | group_counter += 2; |
| 122 | if (group_counter == group_size) { |
| 123 | group_counter = 0; |
| 124 | } |
| 125 | } |
| 126 | } |
| 127 | }; |
| 128 | }; // namespace |
| 129 | |
| 130 | template <typename scalar_t, typename dequantizer_t, typename gemm_t> |
| 131 | void cpu_gemm_wna16_impl( |
| 132 | scalar_t* __restrict__ input, int32_t* __restrict__ q_weight, |
| 133 | scalar_t* __restrict__ output, scalar_t* __restrict__ scales, |
| 134 | int32_t* __restrict__ zeros, scalar_t* __restrict__ bias, |
| 135 | const int32_t m_size, const int32_t n_size, const int32_t k_size, |
| 136 | const int64_t input_stride, const int64_t output_stride, |
| 137 | const int64_t scales_group_stride, const int64_t zeros_group_stride, |
| 138 | const int32_t group_size, const int64_t pack_factor) { |
| 139 | constexpr int32_t gemm_n_tile_size = gemm_t::NSize; |
| 140 | constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize; |
| 141 | constexpr int32_t n_block_size = 16; |
| 142 | static_assert(gemm_n_tile_size % n_block_size == 0); |
| 143 | const int32_t thread_num = cpu_utils::get_max_threads(); |
| 144 | |
| 145 | // a simple schedule policy, just to hold more B tiles in L2 and make sure |
| 146 | // each thread has tasks |
| 147 | const int32_t n_partition_size = [&]() { |
| 148 | const int64_t cache_size = cpu_utils::get_available_l2_size(); |
| 149 | int64_t ps_cache_limit = cache_size / (k_size * sizeof(scalar_t)); |
| 150 | int64_t ps_thread_limit = n_size / thread_num; |
| 151 | ps_cache_limit = |
| 152 | std::max((ps_cache_limit / gemm_n_tile_size) * gemm_n_tile_size, |
| 153 | (int64_t)gemm_n_tile_size); |
| 154 | ps_thread_limit = |
| 155 | std::max((ps_thread_limit / gemm_n_tile_size) * gemm_n_tile_size, |
| 156 | (int64_t)gemm_n_tile_size); |
| 157 | return std::min(ps_cache_limit, ps_thread_limit); |
| 158 | }(); |
| 159 | const int32_t task_num = (n_size + n_partition_size - 1) / n_partition_size; |
| 160 | |
| 161 | // get buffer size |
| 162 | const int64_t b_buffer_size = |
| 163 | (((n_partition_size * k_size * sizeof(scalar_t) + 63) / 64) * 64); |
| 164 | const int64_t c_buffer_size = |
| 165 | (((gemm_m_tile_size * gemm_n_tile_size * sizeof(float) + 63) / 64) * 64); |
| 166 | const int64_t b_buffer_offset = 0; |
| 167 | const int64_t c_buffer_offset = b_buffer_size; |
| 168 | const int64_t buffer_size = b_buffer_size + c_buffer_size; |
| 169 | cpu_utils::ScratchPadManager::get_scratchpad_manager()->realloc(buffer_size * |
| 170 | thread_num); |
| 171 | |
| 172 | alignas(64) cpu_utils::Counter counter; |
| 173 | cpu_utils::Counter* counter_ptr = &counter; |
| 174 | |
| 175 | #pragma omp parallel for schedule(static, 1) |
| 176 | for (int32_t thread_id = 0; thread_id < thread_num; ++thread_id) { |
| 177 | scalar_t* __restrict__ b_buffer = nullptr; |
| 178 | float* __restrict__ c_buffer = nullptr; |
| 179 | { |
| 180 | uint8_t* buffer_ptr = |
| 181 | cpu_utils::ScratchPadManager::get_scratchpad_manager() |
| 182 | ->get_data<uint8_t>() + |
| 183 | thread_id * buffer_size; |
| 184 | b_buffer = reinterpret_cast<scalar_t*>(buffer_ptr + b_buffer_offset); |
| 185 | c_buffer = reinterpret_cast<float*>(buffer_ptr + c_buffer_offset); |
| 186 | } |
| 187 | |
| 188 | const int64_t q_weight_block_stride = n_block_size / pack_factor * k_size; |
| 189 | const int64_t b_buffer_block_stride = n_block_size * k_size; |
| 190 | const int32_t zeros_block_stride = n_block_size / pack_factor; |
| 191 | |
| 192 | gemm_t gemm; |
| 193 | |
| 194 | for (;;) { |
| 195 | int32_t task_id = counter_ptr->acquire_counter(); |
| 196 | |
| 197 | if (task_id >= task_num) { |
| 198 | break; |
| 199 | } |
| 200 | |
| 201 | const int32_t n_start_idx = task_id * n_partition_size; |
| 202 | const int32_t n_block_start_idx = n_start_idx / n_block_size; |
| 203 | const int32_t n_num = std::min(n_partition_size, n_size - n_start_idx); |
| 204 | const int32_t n_block_num = n_num / n_block_size; |
| 205 | // std::printf("thread_id: %d, task_id: %d, n_start_idx: %d, n_num: %d\n", |
| 206 | // thread_id, task_id, n_start_idx, n_num); |
| 207 | |
| 208 | // dequant weight |
| 209 | { |
| 210 | int32_t* __restrict__ curr_q_weight = |
| 211 | q_weight + n_block_start_idx * q_weight_block_stride; |
| 212 | scalar_t* __restrict__ curr_b_buffer = b_buffer; |
| 213 | scalar_t* __restrict__ curr_scales = scales + n_start_idx; |
| 214 | int32_t* __restrict__ curr_zeros = zeros + n_start_idx / pack_factor; |
| 215 | for (int32_t block_idx = 0; block_idx < n_block_num; ++block_idx) { |
| 216 | dequantizer_t::dequant(curr_q_weight, curr_b_buffer, curr_scales, |
| 217 | curr_zeros, scales_group_stride, |
| 218 | zeros_group_stride, k_size, group_size); |
| 219 | |
| 220 | // if (block_idx == 0 && n_start_idx == 0) { |
| 221 | // print_logits("depacked weight", curr_b_buffer, k_size, |
| 222 | // n_block_size, n_block_size); |
| 223 | // } |
| 224 | |
| 225 | // update |
| 226 | curr_q_weight += q_weight_block_stride; |
| 227 | curr_b_buffer += b_buffer_block_stride; |
| 228 | curr_scales += n_block_size; |
| 229 | curr_zeros += zeros_block_stride; |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | // compute loop |
| 234 | { |
| 235 | const int32_t n_tile_num = n_num / gemm_n_tile_size; |
| 236 | scalar_t* __restrict__ curr_input = input; |
| 237 | scalar_t* __restrict__ init_bias = bias; |
| 238 | if (bias != nullptr) { |
| 239 | init_bias += n_start_idx; |
| 240 | } |
| 241 | scalar_t* __restrict__ init_output = output + n_start_idx; |
| 242 | for (int32_t m_idx = 0; m_idx < m_size; m_idx += gemm_m_tile_size) { |
| 243 | const int32_t curr_m_size = |
| 244 | std::min(gemm_m_tile_size, m_size - m_idx); |
| 245 | scalar_t* __restrict__ curr_b_buffer = b_buffer; |
| 246 | scalar_t* __restrict__ curr_bias = init_bias; |
| 247 | scalar_t* __restrict__ curr_output = init_output; |
| 248 | for (int32_t n_tile_idx = 0; n_tile_idx < n_tile_num; ++n_tile_idx) { |
| 249 | gemm.gemm(curr_input, curr_b_buffer, c_buffer, curr_m_size, k_size, |
| 250 | input_stride, b_buffer_block_stride, gemm_n_tile_size, |
| 251 | false); |
| 252 | |
| 253 | if (bias != nullptr) { |
| 254 | cpu_micro_gemm::bias_epilogue<gemm_n_tile_size>( |
| 255 | c_buffer, curr_output, curr_bias, curr_m_size, |
| 256 | gemm_n_tile_size, output_stride); |
| 257 | curr_bias += gemm_n_tile_size; |
| 258 | } else { |
| 259 | cpu_micro_gemm::default_epilogue<gemm_n_tile_size>( |
| 260 | c_buffer, curr_output, curr_m_size, gemm_n_tile_size, |
| 261 | output_stride); |
| 262 | } |
| 263 | |
| 264 | curr_b_buffer += |
| 265 | b_buffer_block_stride * (gemm_n_tile_size / n_block_size); |
| 266 | curr_output += gemm_n_tile_size; |
| 267 | } |
| 268 | curr_input += gemm_m_tile_size * input_stride; |
| 269 | init_output += gemm_m_tile_size * output_stride; |
| 270 | } |
| 271 | } |
| 272 | } |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | void cpu_gemm_wna16( |
| 277 | const torch::Tensor& input, // [M, K] |
| 278 | const torch::Tensor& |
| 279 | q_weight, // [N / 16, K * 16 / pack_factor], packed as int32 |
| 280 | torch::Tensor& output, // [M, N] |
| 281 | const torch::Tensor& scales, // [group_num, N] |
| 282 | const std::optional<torch::Tensor>& |
| 283 | zeros, // [group_num, N / pack_factor], packed as int32 |
| 284 | const std::optional<torch::Tensor>& bias, // [N] |
| 285 | const int64_t pack_factor, const std::string& isa_hint) { |
| 286 | using cpu_utils::ISA; |
| 287 | TORCH_CHECK_EQ(pack_factor, 8); // only supports 4bits |
| 288 | const int32_t a_m_size = input.size(0); |
| 289 | const int32_t a_k_size = input.size(1); |
| 290 | const int64_t a_m_stride = input.stride(0); |
| 291 | const int32_t b_n_size = q_weight.size(0) * 16; |
| 292 | TORCH_CHECK_EQ(a_k_size % 32, 0); |
| 293 | TORCH_CHECK_EQ(b_n_size % 32, 0); |
| 294 | const int32_t group_num = scales.size(0); |
| 295 | const int32_t group_size = a_k_size / group_num; |
| 296 | TORCH_CHECK_EQ(group_size % 2, 0); |
| 297 | const int64_t scales_group_stride = scales.stride(0); |
| 298 | const int64_t output_m_stride = output.stride(0); |
| 299 | |
| 300 | bool has_zp = zeros.has_value(); |
| 301 | |
| 302 | ISA isa = [&]() { |
| 303 | if (isa_hint == "amx") { |
| 304 | return ISA::AMX; |
| 305 | } else if (isa_hint == "vec") { |
| 306 | return ISA::VEC; |
| 307 | } else if (isa_hint == "rvv") { |
| 308 | return ISA::RVV; |
| 309 | } else if (isa_hint == "vsx") { |
| 310 | return ISA::VSX; |
| 311 | } else { |
| 312 | TORCH_CHECK(false, "unsupported isa hint: " + isa_hint); |
| 313 | } |
| 314 | }(); |
| 315 | |
| 316 | int32_t* zeros_ptr = has_zp ? zeros->data_ptr<int32_t>() : nullptr; |
| 317 | const int64_t zeros_group_stride = has_zp ? zeros->stride(0) : 0; |
| 318 | |
| 319 | VLLM_DISPATCH_16B_TYPES(input.scalar_type(), "cpu_gemm_wna16", [&]() { |
| 320 | if (isa == ISA::AMX) { |
| 321 | using gemm_t = cpu_micro_gemm::MicroGemm<ISA::AMX, scalar_t>; |
| 322 | if (has_zp) { |
| 323 | using dequantizer_t = Dequantizer4b<scalar_t, ISA::AMX, true>; |
| 324 | cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>( |
| 325 | input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(), |
| 326 | output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr, |
| 327 | bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size, |
| 328 | b_n_size, a_k_size, a_m_stride, output_m_stride, |
| 329 | scales_group_stride, zeros_group_stride, group_size, pack_factor); |
| 330 | return; |
| 331 | } |
| 332 | { |
| 333 | using dequantizer_t = Dequantizer4b<scalar_t, ISA::AMX, false>; |
| 334 | cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>( |
| 335 | input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(), |
| 336 | output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr, |
| 337 | bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size, |
| 338 | b_n_size, a_k_size, a_m_stride, output_m_stride, |
| 339 | scales_group_stride, zeros_group_stride, group_size, pack_factor); |
| 340 | return; |
| 341 | } |
| 342 | } else if (isa == ISA::VEC) { |
| 343 | using gemm_t = cpu_micro_gemm::MicroGemm<ISA::VEC, scalar_t>; |
| 344 | if (has_zp) { |
| 345 | using dequantizer_t = Dequantizer4b<scalar_t, ISA::VEC, true>; |
| 346 | cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>( |
| 347 | input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(), |
| 348 | output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr, |
| 349 | bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size, |
| 350 | b_n_size, a_k_size, a_m_stride, output_m_stride, |
| 351 | scales_group_stride, zeros_group_stride, group_size, pack_factor); |
| 352 | return; |
| 353 | } |
| 354 | { |
| 355 | using dequantizer_t = Dequantizer4b<scalar_t, ISA::VEC, false>; |
| 356 | cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>( |
| 357 | input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(), |
| 358 | output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr, |
| 359 | bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size, |
| 360 | b_n_size, a_k_size, a_m_stride, output_m_stride, |
| 361 | scales_group_stride, zeros_group_stride, group_size, pack_factor); |
| 362 | return; |
| 363 | } |
| 364 | } else if (isa == ISA::RVV) { |
| 365 | using gemm_t = cpu_micro_gemm::MicroGemm<ISA::RVV, scalar_t>; |
| 366 | if (has_zp) { |
| 367 | using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, true>; |
| 368 | cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>( |
| 369 | input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(), |
| 370 | output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr, |
| 371 | bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size, |
| 372 | b_n_size, a_k_size, a_m_stride, output_m_stride, |
| 373 | scales_group_stride, zeros_group_stride, group_size, pack_factor); |
| 374 | return; |
| 375 | } |
| 376 | { |
| 377 | using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, false>; |
| 378 | cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>( |
| 379 | input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(), |
| 380 | output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr, |
| 381 | bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size, |
| 382 | b_n_size, a_k_size, a_m_stride, output_m_stride, |
| 383 | scales_group_stride, zeros_group_stride, group_size, pack_factor); |
| 384 | return; |
| 385 | } |
| 386 | } else if (isa == ISA::VSX) { |
| 387 | TORCH_CHECK(input.scalar_type() == at::ScalarType::BFloat16, |
| 388 | "VSX WNA16 GEMM only supports BFloat16"); |
| 389 | using gemm_t = cpu_micro_gemm::MicroGemm<ISA::VSX, scalar_t>; |
| 390 | if (has_zp) { |
| 391 | using dequantizer_t = Dequantizer4b<scalar_t, ISA::VSX, true>; |
| 392 | cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>( |
| 393 | input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(), |
| 394 | output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr, |
| 395 | bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size, |
| 396 | b_n_size, a_k_size, a_m_stride, output_m_stride, |
| 397 | scales_group_stride, zeros_group_stride, group_size, pack_factor); |
| 398 | return; |
| 399 | } |
| 400 | { |
| 401 | using dequantizer_t = Dequantizer4b<scalar_t, ISA::VSX, false>; |
| 402 | cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>( |
| 403 | input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(), |
| 404 | output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr, |
| 405 | bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr, a_m_size, |
| 406 | b_n_size, a_k_size, a_m_stride, output_m_stride, |
| 407 | scales_group_stride, zeros_group_stride, group_size, pack_factor); |
| 408 | return; |
| 409 | } |
| 410 | } |
| 411 | }); |
| 412 | } |