[Performance][Hardware][RISC-V] Reduce LMUL pressure in INT4 LUT dequant (#47538)
Signed-off-by: liutong <liutong@iscas.ac.cn> Co-authored-by: Claude <noreply@anthropic.com>
LiuTong committed
Jul 6, 2026 at 13:58 UTC
e433634c781dce7ee55563424b8136ad3495c1be
1 file changed
+23
-10
csrc/cpu/cpu_types_riscv_impl.hpp
+23
-10
@@ -623,17 +623,30 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
623
data.reg, data.reg)) {};
624
explicit FP32Vec16(const FP32Vec16& data) : reg(data.reg) {};
625
explicit FP32Vec16(int64_t value, const FP32Vec16& lut) {
626
- const uint64_t q_values = static_cast<uint64_t>(value);
627
- auto packed = RVVI(__riscv_vmv_v_x_u64, LMUL_1024)(q_values, VEC_ELEM_NUM);
628
- auto lane_ids = RVVI(__riscv_vid_v_u64, LMUL_1024)(VEC_ELEM_NUM);
626
+ // Split into two 32-bit halves to avoid u64 @ LMUL_1024 (m8 on
627
+ // VLEN=128 / m4 on VLEN=256), which causes heavy register spilling.
628
+ constexpr int HALF = VEC_ELEM_NUM / 2;
629
+ const auto q = static_cast<uint64_t>(value);
630
+ const uint32_t lo = static_cast<uint32_t>(q);
631
+ const uint32_t hi = static_cast<uint32_t>(q >> 32);
632
+
633
+ auto lane_ids = RVVI(__riscv_vid_v_u32, LMUL_256)(HALF);
634
auto shifts =
630
- RVVI(__riscv_vsll_vx_u64, LMUL_1024)(lane_ids, 2, VEC_ELEM_NUM);
631
- auto shifted =
632
- RVVI(__riscv_vsrl_vv_u64, LMUL_1024)(packed, shifts, VEC_ELEM_NUM);
633
- auto idx64 =
634
- RVVI(__riscv_vand_vx_u64, LMUL_1024)(shifted, 0xF, VEC_ELEM_NUM);
635
- auto idx32 = RVVI(__riscv_vnsrl_wx_u32, LMUL_512)(idx64, 0, VEC_ELEM_NUM);
636
- reg = RVVI(__riscv_vrgather_vv_f32, LMUL_512)(lut.reg, idx32, VEC_ELEM_NUM);
635
+ RVVI(__riscv_vsll_vx_u32, LMUL_256)(lane_ids, 2, HALF);
636
+
637
+ auto packed_lo = RVVI(__riscv_vmv_v_x_u32, LMUL_256)(lo, HALF);
638
+ auto idx_lo = RVVI(__riscv_vand_vx_u32, LMUL_256)(
639
+ RVVI(__riscv_vsrl_vv_u32, LMUL_256)(packed_lo, shifts, HALF),
640
+ 0xF, HALF);
641
+
642
+ auto packed_hi = RVVI(__riscv_vmv_v_x_u32, LMUL_256)(hi, HALF);
643
+ auto idx_hi = RVVI(__riscv_vand_vx_u32, LMUL_256)(
644
+ RVVI(__riscv_vsrl_vv_u32, LMUL_256)(packed_hi, shifts, HALF),
645
+ 0xF, HALF);
646
+
647
+ auto idx = RVVI4(__riscv_vcreate_v_u32, LMUL_256, _u32,
648
+ LMUL_512)(idx_lo, idx_hi);
649
+ reg = RVVI(__riscv_vrgather_vv_f32, LMUL_512)(lut.reg, idx, VEC_ELEM_NUM);
650
}
651
explicit FP32Vec16(const FP16Vec16& v);
652