target/arm: Implement FMOP4 (non-widening) for float32
Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20260702204314.79224-2-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
Richard Henderson committed
Jul 2, 2026 at 13:43 UTC
0847401e5542f3a4e43c4a5595401179ba5101d6
7 files changed
+139
target/arm/cpu-features.h
+5
@@ -1585,6 +1585,11 @@ static inline bool isar_feature_aa64_ssve_fexpa(const ARMISARegisters *id)
1585
return FIELD_EX64_IDREG(id, ID_AA64SMFR0, SFEXPA);
1586
}
1587
1588
+static inline bool isar_feature_aa64_sme_mop4(const ARMISARegisters *id)
1589
+{
1590
+ return FIELD_EX64_IDREG(id, ID_AA64SMFR0, SMOP4);
1591
+}
1592
+
1593
static inline bool isar_feature_aa64_ssve_aes(const ARMISARegisters *id)
1594
{
1595
return FIELD_EX64_IDREG(id, ID_AA64SMFR0, AES);
target/arm/tcg/helper-sme-defs.h
+4
@@ -355,3 +355,7 @@ DEF_HELPER_FLAGS_5(sme2_sel_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32)
355
DEF_HELPER_FLAGS_5(sme2_sel_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32)
356
DEF_HELPER_FLAGS_5(sme2_sel_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32)
357
DEF_HELPER_FLAGS_5(sme2_sel_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32)
358
+
359
+DEF_HELPER_FLAGS_5(sme_fmop4a_ss, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, fpst, i32)
360
+DEF_HELPER_FLAGS_5(sme_fmop4s_ss, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, fpst, i32)
361
+DEF_HELPER_FLAGS_5(sme_ah_fmop4s_ss, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, fpst, i32)
target/arm/tcg/sme.decode
+12
@@ -1086,3 +1086,15 @@ LUTI4_s_4h 1100 0000 1001 101 idx:1 10 01 00 zn:5 zd:5 &lut
1086
1087
LUTI4_s_4b 1100 0000 1001 101 1 00 00 00 ....0 zd:5 \
1088
&lut zn=%zn_ax2 idx=0
1089
+
1090
+# SME MOP4 Quarter-tile outer products
1091
+
1092
+&mop4 zad zn zm s:bool n:bool m:bool
1093
+
1094
+%mop4_zm 17:3 !function=times_2_plus_16
1095
+%mop4_zn 6:3 !function=times_2
1096
+
1097
+@mop4_o2 .... .... ... m:1 .... .... .. n:1 ... . s:1 .. zad:2 \
1098
+ &mop4 zm=%mop4_zm zn=%mop4_zn
1099
+
1100
+FMOP4_ss 1000 0000 000. ...0 0000 00.. ..0. 00.. @mop4_o2
target/arm/tcg/sme_helper.c
+76
@@ -2601,3 +2601,79 @@ void HELPER(sme2_sel_d)(void *vd, void *vn, void *vm,
2601
}
2602
}
2603
}
2604
+
2605
+void sme_mop4(void *vza, void *vzn, void *vzm, void *fn_opaque,
2606
+ uint32_t desc, size_t esize,
2607
+ void (*fn)(void *, void *, void *, void *))
2608
+{
2609
+ intptr_t oprsz = simd_maxsz(desc);
2610
+ intptr_t dim = oprsz / 2; /* in bytes */
2611
+ bool nreg_m1 = extract32(desc, SIMD_DATA_SHIFT + 0, 1);
2612
+ bool mreg_m1 = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
2613
+ intptr_t host_adj = HOST_BIG_ENDIAN ? 8 - esize : 0;
2614
+
2615
+ for (int outprod = 0; outprod < 4; outprod++) {
2616
+ bool row_hv = outprod & 2;
2617
+ bool col_hv = outprod & 1;
2618
+ intptr_t row_base = row_hv ? dim : 0;
2619
+ intptr_t col_base = col_hv ? dim : 0;
2620
+ void *op1 = vzn + (col_hv && nreg_m1 ? sizeof(ARMVectorReg) : 0);
2621
+ void *op2 = vzm + (row_hv && mreg_m1 ? sizeof(ARMVectorReg) : 0);
2622
+
2623
+ for (intptr_t row = 0; row < dim; row += esize) {
2624
+ intptr_t row_idx = row_base + row;
2625
+ void *vza_row = vza + tile_vslice_offset(row_idx);
2626
+ void *e1 = op1 + (row_idx ^ host_adj);
2627
+
2628
+ for (intptr_t col = 0; col < dim; col += esize) {
2629
+ intptr_t col_idx = col_base + col;
2630
+ void *e2 = op2 + (col_idx ^ host_adj);
2631
+ void *e3 = vza_row + (col_idx ^ host_adj);
2632
+
2633
+ fn(e3, e1, e2, fn_opaque);
2634
+ }
2635
+ }
2636
+ }
2637
+}
2638
+
2639
+static void inner_fmop4a_ss(void *vd, void *vn, void *vm, void *vinfo)
2640
+{
2641
+ float32 *d = vd, *n = vn, *m = vm;
2642
+ float_status *fpst = vinfo;
2643
+
2644
+ *d = float32_muladd(*n, *m, *d, 0, fpst);
2645
+}
2646
+
2647
+void HELPER(sme_fmop4a_ss)(void *vza, void *vzn, void *vzm,
2648
+ float_status *fpst, uint32_t desc)
2649
+{
2650
+ sme_mop4(vza, vzn, vzm, fpst, desc, sizeof(float32), inner_fmop4a_ss);
2651
+}
2652
+
2653
+static void inner_fmop4s_ss(void *vd, void *vn, void *vm, void *vinfo)
2654
+{
2655
+ float32 *d = vd, *n = vn, *m = vm;
2656
+ float_status *fpst = vinfo;
2657
+
2658
+ *d = float32_muladd(float32_chs(*n), *m, *d, 0, fpst);
2659
+}
2660
+
2661
+void HELPER(sme_fmop4s_ss)(void *vza, void *vzn, void *vzm,
2662
+ float_status *fpst, uint32_t desc)
2663
+{
2664
+ sme_mop4(vza, vzn, vzm, fpst, desc, sizeof(float32), inner_fmop4s_ss);
2665
+}
2666
+
2667
+static void inner_ah_fmop4s_ss(void *vd, void *vn, void *vm, void *vinfo)
2668
+{
2669
+ float32 *d = vd, *n = vn, *m = vm;
2670
+ float_status *fpst = vinfo;
2671
+
2672
+ *d = float32_muladd(*n, *m, *d, float_muladd_negate_product, fpst);
2673
+}
2674
+
2675
+void HELPER(sme_ah_fmop4s_ss)(void *vza, void *vzn, void *vzm,
2676
+ float_status *fpst, uint32_t desc)
2677
+{
2678
+ sme_mop4(vza, vzn, vzm, fpst, desc, sizeof(float32), inner_ah_fmop4s_ss);
2679
+}
target/arm/tcg/translate-sme.c
+29
@@ -2002,3 +2002,32 @@ TRANS_FEAT(LUTI4_s_4h, aa64_sme2p1, do_lut_s4, a, gen_helper_sme2_luti4_4h)
2002
2003
TRANS_FEAT(LUTI4_s_4b, aa64_sme2p1_lutv2, do_lut_s4, a,
2004
gen_helper_sme2_luti4_4b)
2005
+
2006
+static bool do_mop4_fp(DisasContext *s, arg_mop4 *a, MemOp esz,
2007
+ ARMFPStatusFlavour e_fpst,
2008
+ gen_helper_gvec_3_ptr * const fns[3])
2009
+{
2010
+ int svl = streaming_vec_reg_size(s);
2011
+ uint32_t desc = simd_desc(svl, svl, (a->m << 1) | a->n);
2012
+ int fns_idx = (a->s ? 1 + s->fpcr_ah : 0);
2013
+ TCGv_ptr za, zn, zm, fpst;
2014
+
2015
+ if (!sme_smza_enabled_check(s)) {
2016
+ return true;
2017
+ }
2018
+
2019
+ za = get_tile(s, esz, a->zad);
2020
+ zn = vec_full_reg_ptr(s, a->zn);
2021
+ zm = vec_full_reg_ptr(s, a->zm);
2022
+ fpst = fpstatus_ptr(e_fpst);
2023
+
2024
+ fns[fns_idx](za, zn, zm, fpst, tcg_constant_i32(desc));
2025
+ return true;
2026
+}
2027
+
2028
+static gen_helper_gvec_3_ptr * const fmop4_ss[3] = {
2029
+ gen_helper_sme_fmop4a_ss,
2030
+ gen_helper_sme_fmop4s_ss,
2031
+ gen_helper_sme_ah_fmop4s_ss
2032
+};
2033
+TRANS_FEAT(FMOP4_ss, aa64_sme_mop4, do_mop4_fp, a, MO_32, FPST_ZA, fmop4_ss)
target/arm/tcg/translate.h
+5
@@ -268,6 +268,11 @@ static inline int times_2_plus_1(DisasContext *s, int x)
268
return x * 2 + 1;
269
}
270
271
+static inline int times_2_plus_16(DisasContext *s, int x)
272
+{
273
+ return x * 2 + 16;
274
+}
275
+
276
static inline int rsub_64(DisasContext *s, int x)
277
{
278
return 64 - x;
target/arm/tcg/vec_internal.h
+8
@@ -547,4 +547,12 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, \
547
clear_tail(d, oprsz, simd_maxsz(desc)); \
548
}
549
550
+/*
551
+ * Perform SME quarter-tile outer product.
552
+ * Iterate over ZAtile[] for esize, calling fn for each element.
553
+ */
554
+void sme_mop4(void *vza, void *vzn, void *vzm, void *fn_opaque,
555
+ uint32_t desc, size_t esize,
556
+ void (*fn)(void *, void *, void *, void *));
557
+
558
#endif /* TARGET_ARM_VEC_INTERNAL_H */