| 1 | /* |
| 2 | * ARM AdvSIMD / SVE Vector Operations |
| 3 | * |
| 4 | * Copyright (c) 2026 Linaro |
| 5 | * |
| 6 | * SPDX-License-Identifier: GPL-2.0-or-later |
| 7 | */ |
| 8 | |
| 9 | #include "qemu/osdep.h" |
| 10 | #include "cpu.h" |
| 11 | #include "internals.h" |
| 12 | #include "helper.h" |
| 13 | #include "helper-a64.h" |
| 14 | #include "helper-sme.h" |
| 15 | #include "helper-sve.h" |
| 16 | #include "tcg/tcg-gvec-desc.h" |
| 17 | #include "fpu/softfloat.h" |
| 18 | #include "qemu/int128.h" |
| 19 | #include "crypto/clmul.h" |
| 20 | #include "vec_internal.h" |
| 21 | |
| 22 | DO_3OP(gvec_fdiv_h, float16_div, float16) |
| 23 | DO_3OP(gvec_fdiv_s, float32_div, float32) |
| 24 | DO_3OP(gvec_fdiv_d, float64_div, float64) |
| 25 | |
| 26 | DO_3OP(gvec_fmulx_h, helper_advsimd_mulxh, float16) |
| 27 | DO_3OP(gvec_fmulx_s, helper_vfp_mulxs, float32) |
| 28 | DO_3OP(gvec_fmulx_d, helper_vfp_mulxd, float64) |
| 29 | |
| 30 | DO_3OP(gvec_recps_h, helper_recpsf_f16, float16) |
| 31 | DO_3OP(gvec_recps_s, helper_recpsf_f32, float32) |
| 32 | DO_3OP(gvec_recps_d, helper_recpsf_f64, float64) |
| 33 | |
| 34 | DO_3OP(gvec_rsqrts_h, helper_rsqrtsf_f16, float16) |
| 35 | DO_3OP(gvec_rsqrts_s, helper_rsqrtsf_f32, float32) |
| 36 | DO_3OP(gvec_rsqrts_d, helper_rsqrtsf_f64, float64) |
| 37 | |
| 38 | DO_3OP(gvec_ah_recps_h, helper_recpsf_ah_f16, float16) |
| 39 | DO_3OP(gvec_ah_recps_s, helper_recpsf_ah_f32, float32) |
| 40 | DO_3OP(gvec_ah_recps_d, helper_recpsf_ah_f64, float64) |
| 41 | |
| 42 | DO_3OP(gvec_ah_rsqrts_h, helper_rsqrtsf_ah_f16, float16) |
| 43 | DO_3OP(gvec_ah_rsqrts_s, helper_rsqrtsf_ah_f32, float32) |
| 44 | DO_3OP(gvec_ah_rsqrts_d, helper_rsqrtsf_ah_f64, float64) |
| 45 | |
| 46 | DO_3OP(gvec_ah_fmax_h, helper_vfp_ah_maxh, float16) |
| 47 | DO_3OP(gvec_ah_fmax_s, helper_vfp_ah_maxs, float32) |
| 48 | DO_3OP(gvec_ah_fmax_d, helper_vfp_ah_maxd, float64) |
| 49 | |
| 50 | DO_3OP(gvec_ah_fmin_h, helper_vfp_ah_minh, float16) |
| 51 | DO_3OP(gvec_ah_fmin_s, helper_vfp_ah_mins, float32) |
| 52 | DO_3OP(gvec_ah_fmin_d, helper_vfp_ah_mind, float64) |
| 53 | |
| 54 | DO_3OP(gvec_fmax_b16, bfloat16_max, bfloat16) |
| 55 | DO_3OP(gvec_fmin_b16, bfloat16_min, bfloat16) |
| 56 | DO_3OP(gvec_fmaxnum_b16, bfloat16_maxnum, bfloat16) |
| 57 | DO_3OP(gvec_fminnum_b16, bfloat16_minnum, bfloat16) |
| 58 | DO_3OP(gvec_ah_fmax_b16, helper_sme2_ah_fmax_b16, bfloat16) |
| 59 | DO_3OP(gvec_ah_fmin_b16, helper_sme2_ah_fmin_b16, bfloat16) |
| 60 | |
| 61 | #define nop(N, M, S) (M) |
| 62 | |
| 63 | DO_FMUL_IDX(gvec_fmulx_idx_h, nop, helper_advsimd_mulxh, float16, H2) |
| 64 | DO_FMUL_IDX(gvec_fmulx_idx_s, nop, helper_vfp_mulxs, float32, H4) |
| 65 | DO_FMUL_IDX(gvec_fmulx_idx_d, nop, helper_vfp_mulxd, float64, H8) |
| 66 | |
| 67 | #undef nop |
| 68 | |
| 69 | void HELPER(sve2_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc) |
| 70 | { |
| 71 | int shift = simd_data(desc) * 8; |
| 72 | intptr_t i, opr_sz = simd_oprsz(desc); |
| 73 | uint64_t *d = vd, *n = vn, *m = vm; |
| 74 | |
| 75 | for (i = 0; i < opr_sz / 8; ++i) { |
| 76 | d[i] = clmul_8x4_even(n[i] >> shift, m[i] >> shift); |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc) |
| 81 | { |
| 82 | intptr_t sel = H4(simd_data(desc)); |
| 83 | intptr_t i, opr_sz = simd_oprsz(desc); |
| 84 | uint32_t *n = vn, *m = vm; |
| 85 | uint64_t *d = vd; |
| 86 | |
| 87 | for (i = 0; i < opr_sz / 8; ++i) { |
| 88 | d[i] = clmul_32(n[2 * i + sel], m[2 * i + sel]); |
| 89 | } |
| 90 | } |
| 91 | |
| 92 | void HELPER(sve_pmull_q)(void *vd, void *vn, void *vm, uint32_t desc) |
| 93 | { |
| 94 | intptr_t opr_sz = simd_oprsz(desc); |
| 95 | uint64_t *n = vn, *m = vm; |
| 96 | uint64_t *d0 = vd; |
| 97 | uint64_t *d1 = vd + sizeof(ARMVectorReg); |
| 98 | |
| 99 | for (intptr_t i = 0; i < opr_sz / 16; ++i) { |
| 100 | Int128 rl = clmul_64(n[2 * i + 0], m[2 * i + 0]); |
| 101 | Int128 rh = clmul_64(n[2 * i + 1], m[2 * i + 1]); |
| 102 | d0[2 * i + 0] = int128_getlo(rl); |
| 103 | d0[2 * i + 1] = int128_gethi(rl); |
| 104 | d1[2 * i + 0] = int128_getlo(rh); |
| 105 | d1[2 * i + 1] = int128_gethi(rh); |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | void HELPER(sve_pmlal_q)(void *vd, void *vn, void *vm, uint32_t desc) |
| 110 | { |
| 111 | intptr_t opr_sz = simd_oprsz(desc); |
| 112 | uint64_t *n = vn, *m = vm; |
| 113 | uint64_t *d0 = vd; |
| 114 | uint64_t *d1 = vd + sizeof(ARMVectorReg); |
| 115 | |
| 116 | for (intptr_t i = 0; i < opr_sz / 16; ++i) { |
| 117 | Int128 rl = clmul_64(n[2 * i + 0], m[2 * i + 0]); |
| 118 | Int128 rh = clmul_64(n[2 * i + 1], m[2 * i + 1]); |
| 119 | d0[2 * i + 0] ^= int128_getlo(rl); |
| 120 | d0[2 * i + 1] ^= int128_gethi(rl); |
| 121 | d1[2 * i + 0] ^= int128_getlo(rh); |
| 122 | d1[2 * i + 1] ^= int128_gethi(rh); |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | DO_3OP_PAIR(gvec_ah_fmaxp_h, helper_vfp_ah_maxh, float16, H2) |
| 127 | DO_3OP_PAIR(gvec_ah_fmaxp_s, helper_vfp_ah_maxs, float32, H4) |
| 128 | DO_3OP_PAIR(gvec_ah_fmaxp_d, helper_vfp_ah_maxd, float64, /**/) |
| 129 | |
| 130 | DO_3OP_PAIR(gvec_ah_fminp_h, helper_vfp_ah_minh, float16, H2) |
| 131 | DO_3OP_PAIR(gvec_ah_fminp_s, helper_vfp_ah_mins, float32, H4) |
| 132 | DO_3OP_PAIR(gvec_ah_fminp_d, helper_vfp_ah_mind, float64, /**/) |
| 133 | |
| 134 | void HELPER(simd_tblx)(void *vd, void *vm, CPUARMState *env, uint32_t desc) |
| 135 | { |
| 136 | const uint8_t *indices = vm; |
| 137 | size_t oprsz = simd_oprsz(desc); |
| 138 | uint32_t rn = extract32(desc, SIMD_DATA_SHIFT, 5); |
| 139 | bool is_tbx = extract32(desc, SIMD_DATA_SHIFT + 5, 1); |
| 140 | uint32_t table_len = desc >> (SIMD_DATA_SHIFT + 6); |
| 141 | union { |
| 142 | uint8_t b[16]; |
| 143 | uint64_t d[2]; |
| 144 | } result; |
| 145 | |
| 146 | /* |
| 147 | * We must construct the final result in a temp, lest the output |
| 148 | * overlaps the input table. For TBL, begin with zero; for TBX, |
| 149 | * begin with the original register contents. Note that we always |
| 150 | * copy 16 bytes here to avoid an extra branch; clearing the high |
| 151 | * bits of the register for oprsz == 8 is handled below. |
| 152 | */ |
| 153 | if (is_tbx) { |
| 154 | memcpy(&result, vd, 16); |
| 155 | } else { |
| 156 | memset(&result, 0, 16); |
| 157 | } |
| 158 | |
| 159 | for (size_t i = 0; i < oprsz; ++i) { |
| 160 | uint32_t index = indices[H1(i)]; |
| 161 | |
| 162 | if (index < table_len) { |
| 163 | /* |
| 164 | * Convert index (a byte offset into the virtual table |
| 165 | * which is a series of 128-bit vectors concatenated) |
| 166 | * into the correct register element, bearing in mind |
| 167 | * that the table can wrap around from V31 to V0. |
| 168 | */ |
| 169 | const uint8_t *table = (const uint8_t *) |
| 170 | aa64_vfp_qreg(env, (rn + (index >> 4)) % 32); |
| 171 | result.b[H1(i)] = table[H1(index % 16)]; |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | memcpy(vd, &result, 16); |
| 176 | clear_tail(vd, oprsz, simd_maxsz(desc)); |
| 177 | } |
| 178 | |
| 179 | /* |
| 180 | * Use float_minmax_ismag to get the absolute value min/max. |
| 181 | * Avoid float_minmax_is{num,number} so that we get normal NaN processing. |
| 182 | * If the result is not a nan, take the absolute value. |
| 183 | * |
| 184 | * Note this operation squashes FZ, FIZ, and AH to 0. |
| 185 | */ |
| 186 | #define DO_FAMINMAX(NAME, TYPE, MIN) \ |
| 187 | TYPE TYPE##_##NAME(TYPE a, TYPE b, float_status *s) \ |
| 188 | { \ |
| 189 | float_status local = *s; \ |
| 190 | set_flush_to_zero(false, &local); \ |
| 191 | set_flush_inputs_to_zero(false, &local); \ |
| 192 | arm_set_default_fp_behaviours(&local); \ |
| 193 | TYPE r = TYPE##_minmax(a, b, &local, MIN | float_minmax_ismag); \ |
| 194 | if (!TYPE##_is_any_nan(r)) { \ |
| 195 | r = TYPE##_abs(r); \ |
| 196 | } \ |
| 197 | float_raise(get_float_exception_flags(&local) \ |
| 198 | & ~float_flag_input_denormal_used, s); \ |
| 199 | return r; \ |
| 200 | } |
| 201 | |
| 202 | DO_FAMINMAX(famax, float16, 0) |
| 203 | DO_FAMINMAX(famin, float16, float_minmax_ismin) |
| 204 | DO_FAMINMAX(famax, float32, 0) |
| 205 | DO_FAMINMAX(famin, float32, float_minmax_ismin) |
| 206 | DO_FAMINMAX(famax, float64, 0) |
| 207 | DO_FAMINMAX(famin, float64, float_minmax_ismin) |
| 208 | |
| 209 | DO_3OP(gvec_famax_h, float16_famax, float16) |
| 210 | DO_3OP(gvec_famin_h, float16_famin, float16) |
| 211 | DO_3OP(gvec_famax_s, float32_famax, float32) |
| 212 | DO_3OP(gvec_famin_s, float32_famin, float32) |
| 213 | DO_3OP(gvec_famax_d, float64_famax, float64) |
| 214 | DO_3OP(gvec_famin_d, float64_famin, float64) |
| 215 | |
| 216 | DO_3OP(gvec_fscale_b16, bfloat16_scalbn, int16_t) |
| 217 | DO_3OP(gvec_fscale_h, float16_scalbn, int16_t) |
| 218 | DO_3OP(gvec_fscale_s, float32_scalbn, int32_t) |
| 219 | DO_3OP(gvec_fscale_d, scalbn_d, int64_t) |