target/arm/tcg/vec_helper.c: make compilation unit common
We need to extract 64 bits helper in a new file (vec_helper64.c), and extract some macro definition also, since they will be used in both files. As well, DO_3OP_PAIR was defined twice, so rename the second variant to DO_3OP_PAIR_NO_STATUS to reflect what it does. Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Message-id: 20260219040150.2098396-12-pierrick.bouvier@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
Pierrick Bouvier committed
Feb 18, 2026 at 20:01 UTC
4ba9234f67f296beb998b7d7f77d70722d5fd5c1
4 files changed
+212
-208
target/arm/tcg/meson.build
+3
-1
@@ -33,7 +33,6 @@ arm_ss.add(files(
33
'm_helper.c',
34
'mve_helper.c',
35
'op_helper.c',
36
- 'vec_helper.c',
36
))
37
38
arm_ss.add(when: 'TARGET_AARCH64', if_true: files(
@@ -47,6 +46,7 @@ arm_ss.add(when: 'TARGET_AARCH64', if_true: files(
46
'pauth_helper.c',
47
'sme_helper.c',
48
'sve_helper.c',
49
+ 'vec_helper64.c',
50
))
51
52
arm_common_system_ss.add(when: 'CONFIG_ARM_V7M', if_true: files('cpu-v7m.c'))
@@ -67,6 +67,7 @@ arm_common_system_ss.add(files(
67
'psci.c',
68
'tlb_helper.c',
69
'tlb-insns.c',
70
+ 'vec_helper.c',
71
'vfp_helper.c',
72
))
73
arm_user_ss.add(files(
@@ -74,5 +75,6 @@ arm_user_ss.add(files(
75
'hflags.c',
76
'neon_helper.c',
77
'tlb_helper.c',
78
+ 'vec_helper.c',
79
'vfp_helper.c',
80
))
target/arm/tcg/vec_helper.c
+18
-207
@@ -20,9 +20,6 @@
20
#include "qemu/osdep.h"
21
#include "cpu.h"
22
#include "helper.h"
23
-#include "helper-a64.h"
24
-#include "helper-sme.h"
25
-#include "helper-sve.h"
23
#include "tcg/tcg-gvec-desc.h"
24
#include "fpu/softfloat.h"
25
#include "qemu/int128.h"
@@ -1458,18 +1455,6 @@ static float32 float32_rsqrts_nf(float32 op1, float32 op2, float_status *stat)
1455
return float32_div(op1, float32_two, stat);
1456
}
1457
1461
-#define DO_3OP(NAME, FUNC, TYPE) \
1462
-void HELPER(NAME)(void *vd, void *vn, void *vm, \
1463
- float_status *stat, uint32_t desc) \
1464
-{ \
1465
- intptr_t i, oprsz = simd_oprsz(desc); \
1466
- TYPE *d = vd, *n = vn, *m = vm; \
1467
- for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1468
- d[i] = FUNC(n[i], m[i], stat); \
1469
- } \
1470
- clear_tail(d, oprsz, simd_maxsz(desc)); \
1471
-}
1472
-
1458
DO_3OP(gvec_fadd_b16, bfloat16_add, float16)
1459
DO_3OP(gvec_fadd_h, float16_add, float16)
1460
DO_3OP(gvec_fadd_s, float32_add, float32)
@@ -1541,49 +1526,6 @@ DO_3OP(gvec_recps_nf_s, float32_recps_nf, float32)
1526
DO_3OP(gvec_rsqrts_nf_h, float16_rsqrts_nf, float16)
1527
DO_3OP(gvec_rsqrts_nf_s, float32_rsqrts_nf, float32)
1528
1544
-#ifdef TARGET_AARCH64
1545
-DO_3OP(gvec_fdiv_h, float16_div, float16)
1546
-DO_3OP(gvec_fdiv_s, float32_div, float32)
1547
-DO_3OP(gvec_fdiv_d, float64_div, float64)
1548
-
1549
-DO_3OP(gvec_fmulx_h, helper_advsimd_mulxh, float16)
1550
-DO_3OP(gvec_fmulx_s, helper_vfp_mulxs, float32)
1551
-DO_3OP(gvec_fmulx_d, helper_vfp_mulxd, float64)
1552
-
1553
-DO_3OP(gvec_recps_h, helper_recpsf_f16, float16)
1554
-DO_3OP(gvec_recps_s, helper_recpsf_f32, float32)
1555
-DO_3OP(gvec_recps_d, helper_recpsf_f64, float64)
1556
-
1557
-DO_3OP(gvec_rsqrts_h, helper_rsqrtsf_f16, float16)
1558
-DO_3OP(gvec_rsqrts_s, helper_rsqrtsf_f32, float32)
1559
-DO_3OP(gvec_rsqrts_d, helper_rsqrtsf_f64, float64)
1560
-
1561
-DO_3OP(gvec_ah_recps_h, helper_recpsf_ah_f16, float16)
1562
-DO_3OP(gvec_ah_recps_s, helper_recpsf_ah_f32, float32)
1563
-DO_3OP(gvec_ah_recps_d, helper_recpsf_ah_f64, float64)
1564
-
1565
-DO_3OP(gvec_ah_rsqrts_h, helper_rsqrtsf_ah_f16, float16)
1566
-DO_3OP(gvec_ah_rsqrts_s, helper_rsqrtsf_ah_f32, float32)
1567
-DO_3OP(gvec_ah_rsqrts_d, helper_rsqrtsf_ah_f64, float64)
1568
-
1569
-DO_3OP(gvec_ah_fmax_h, helper_vfp_ah_maxh, float16)
1570
-DO_3OP(gvec_ah_fmax_s, helper_vfp_ah_maxs, float32)
1571
-DO_3OP(gvec_ah_fmax_d, helper_vfp_ah_maxd, float64)
1572
-
1573
-DO_3OP(gvec_ah_fmin_h, helper_vfp_ah_minh, float16)
1574
-DO_3OP(gvec_ah_fmin_s, helper_vfp_ah_mins, float32)
1575
-DO_3OP(gvec_ah_fmin_d, helper_vfp_ah_mind, float64)
1576
-
1577
-DO_3OP(gvec_fmax_b16, bfloat16_max, bfloat16)
1578
-DO_3OP(gvec_fmin_b16, bfloat16_min, bfloat16)
1579
-DO_3OP(gvec_fmaxnum_b16, bfloat16_maxnum, bfloat16)
1580
-DO_3OP(gvec_fminnum_b16, bfloat16_minnum, bfloat16)
1581
-DO_3OP(gvec_ah_fmax_b16, helper_sme2_ah_fmax_b16, bfloat16)
1582
-DO_3OP(gvec_ah_fmin_b16, helper_sme2_ah_fmin_b16, bfloat16)
1583
-
1584
-#endif
1585
-#undef DO_3OP
1586
-
1529
/* Non-fused multiply-add (unlike float16_muladd etc, which are fused) */
1530
static float16 float16_muladd_nf(float16 dest, float16 op1, float16 op2,
1531
float_status *stat)
@@ -1769,23 +1711,6 @@ DO_MLA_IDX(gvec_mls_idx_d, uint64_t, -, H8)
1711
1712
#undef DO_MLA_IDX
1713
1772
-#define DO_FMUL_IDX(NAME, ADD, MUL, TYPE, H) \
1773
-void HELPER(NAME)(void *vd, void *vn, void *vm, \
1774
- float_status *stat, uint32_t desc) \
1775
-{ \
1776
- intptr_t i, j, oprsz = simd_oprsz(desc); \
1777
- intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
1778
- intptr_t idx = simd_data(desc); \
1779
- TYPE *d = vd, *n = vn, *m = vm; \
1780
- for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1781
- TYPE mm = m[H(i + idx)]; \
1782
- for (j = 0; j < segment; j++) { \
1783
- d[i + j] = ADD(d[i + j], MUL(n[i + j], mm, stat), stat); \
1784
- } \
1785
- } \
1786
- clear_tail(d, oprsz, simd_maxsz(desc)); \
1787
-}
1788
-
1714
#define nop(N, M, S) (M)
1715
1716
DO_FMUL_IDX(gvec_fmul_idx_b16, nop, bfloat16_mul, float16, H2)
@@ -1793,14 +1718,6 @@ DO_FMUL_IDX(gvec_fmul_idx_h, nop, float16_mul, float16, H2)
1718
DO_FMUL_IDX(gvec_fmul_idx_s, nop, float32_mul, float32, H4)
1719
DO_FMUL_IDX(gvec_fmul_idx_d, nop, float64_mul, float64, H8)
1720
1796
-#ifdef TARGET_AARCH64
1797
-
1798
-DO_FMUL_IDX(gvec_fmulx_idx_h, nop, helper_advsimd_mulxh, float16, H2)
1799
-DO_FMUL_IDX(gvec_fmulx_idx_s, nop, helper_vfp_mulxs, float32, H4)
1800
-DO_FMUL_IDX(gvec_fmulx_idx_d, nop, helper_vfp_mulxd, float64, H8)
1801
-
1802
-#endif
1803
-
1721
#undef nop
1722
1723
/*
@@ -1812,8 +1729,6 @@ DO_FMUL_IDX(gvec_fmla_nf_idx_s, float32_add, float32_mul, float32, H4)
1729
DO_FMUL_IDX(gvec_fmls_nf_idx_h, float16_sub, float16_mul, float16, H2)
1730
DO_FMUL_IDX(gvec_fmls_nf_idx_s, float32_sub, float32_mul, float32, H4)
1731
1815
-#undef DO_FMUL_IDX
1816
-
1732
#define DO_FMLA_IDX(NAME, TYPE, H, NEGX, NEGF) \
1733
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, \
1734
float_status *stat, uint32_t desc) \
@@ -2530,31 +2445,6 @@ void HELPER(neon_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc)
2445
clear_tail(d, 16, simd_maxsz(desc));
2446
}
2447
2533
-#ifdef TARGET_AARCH64
2534
-void HELPER(sve2_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc)
2535
-{
2536
- int shift = simd_data(desc) * 8;
2537
- intptr_t i, opr_sz = simd_oprsz(desc);
2538
- uint64_t *d = vd, *n = vn, *m = vm;
2539
-
2540
- for (i = 0; i < opr_sz / 8; ++i) {
2541
- d[i] = clmul_8x4_even(n[i] >> shift, m[i] >> shift);
2542
- }
2543
-}
2544
-
2545
-void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc)
2546
-{
2547
- intptr_t sel = H4(simd_data(desc));
2548
- intptr_t i, opr_sz = simd_oprsz(desc);
2549
- uint32_t *n = vn, *m = vm;
2550
- uint64_t *d = vd;
2551
-
2552
- for (i = 0; i < opr_sz / 8; ++i) {
2553
- d[i] = clmul_32(n[2 * i + sel], m[2 * i + sel]);
2554
- }
2555
-}
2556
-#endif
2557
-
2448
#define DO_CMP0(NAME, TYPE, OP) \
2449
void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
2450
{ \
@@ -2628,26 +2518,6 @@ DO_ABA(gvec_uaba_d, uint64_t)
2518
2519
#undef DO_ABA
2520
2631
-#define DO_3OP_PAIR(NAME, FUNC, TYPE, H) \
2632
-void HELPER(NAME)(void *vd, void *vn, void *vm, \
2633
- float_status *stat, uint32_t desc) \
2634
-{ \
2635
- ARMVectorReg scratch; \
2636
- intptr_t oprsz = simd_oprsz(desc); \
2637
- intptr_t half = oprsz / sizeof(TYPE) / 2; \
2638
- TYPE *d = vd, *n = vn, *m = vm; \
2639
- if (unlikely(d == m)) { \
2640
- m = memcpy(&scratch, m, oprsz); \
2641
- } \
2642
- for (intptr_t i = 0; i < half; ++i) { \
2643
- d[H(i)] = FUNC(n[H(i * 2)], n[H(i * 2 + 1)], stat); \
2644
- } \
2645
- for (intptr_t i = 0; i < half; ++i) { \
2646
- d[H(i + half)] = FUNC(m[H(i * 2)], m[H(i * 2 + 1)], stat); \
2647
- } \
2648
- clear_tail(d, oprsz, simd_maxsz(desc)); \
2649
-}
2650
-
2521
DO_3OP_PAIR(gvec_faddp_h, float16_add, float16, H2)
2522
DO_3OP_PAIR(gvec_faddp_s, float32_add, float32, H4)
2523
DO_3OP_PAIR(gvec_faddp_d, float64_add, float64, )
@@ -2668,19 +2538,7 @@ DO_3OP_PAIR(gvec_fminnump_h, float16_minnum, float16, H2)
2538
DO_3OP_PAIR(gvec_fminnump_s, float32_minnum, float32, H4)
2539
DO_3OP_PAIR(gvec_fminnump_d, float64_minnum, float64, )
2540
2671
-#ifdef TARGET_AARCH64
2672
-DO_3OP_PAIR(gvec_ah_fmaxp_h, helper_vfp_ah_maxh, float16, H2)
2673
-DO_3OP_PAIR(gvec_ah_fmaxp_s, helper_vfp_ah_maxs, float32, H4)
2674
-DO_3OP_PAIR(gvec_ah_fmaxp_d, helper_vfp_ah_maxd, float64, )
2675
-
2676
-DO_3OP_PAIR(gvec_ah_fminp_h, helper_vfp_ah_minh, float16, H2)
2677
-DO_3OP_PAIR(gvec_ah_fminp_s, helper_vfp_ah_mins, float32, H4)
2678
-DO_3OP_PAIR(gvec_ah_fminp_d, helper_vfp_ah_mind, float64, )
2679
-#endif
2680
-
2681
-#undef DO_3OP_PAIR
2682
-
2683
-#define DO_3OP_PAIR(NAME, FUNC, TYPE, H) \
2541
+#define DO_3OP_PAIR_NO_STATUS(NAME, FUNC, TYPE, H) \
2542
void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
2543
{ \
2544
ARMVectorReg scratch; \
@@ -2700,29 +2558,29 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
2558
}
2559
2560
#define ADD(A, B) (A + B)
2703
-DO_3OP_PAIR(gvec_addp_b, ADD, uint8_t, H1)
2704
-DO_3OP_PAIR(gvec_addp_h, ADD, uint16_t, H2)
2705
-DO_3OP_PAIR(gvec_addp_s, ADD, uint32_t, H4)
2706
-DO_3OP_PAIR(gvec_addp_d, ADD, uint64_t, )
2561
+DO_3OP_PAIR_NO_STATUS(gvec_addp_b, ADD, uint8_t, H1)
2562
+DO_3OP_PAIR_NO_STATUS(gvec_addp_h, ADD, uint16_t, H2)
2563
+DO_3OP_PAIR_NO_STATUS(gvec_addp_s, ADD, uint32_t, H4)
2564
+DO_3OP_PAIR_NO_STATUS(gvec_addp_d, ADD, uint64_t, /**/)
2565
#undef ADD
2566
2709
-DO_3OP_PAIR(gvec_smaxp_b, MAX, int8_t, H1)
2710
-DO_3OP_PAIR(gvec_smaxp_h, MAX, int16_t, H2)
2711
-DO_3OP_PAIR(gvec_smaxp_s, MAX, int32_t, H4)
2567
+DO_3OP_PAIR_NO_STATUS(gvec_smaxp_b, MAX, int8_t, H1)
2568
+DO_3OP_PAIR_NO_STATUS(gvec_smaxp_h, MAX, int16_t, H2)
2569
+DO_3OP_PAIR_NO_STATUS(gvec_smaxp_s, MAX, int32_t, H4)
2570
2713
-DO_3OP_PAIR(gvec_umaxp_b, MAX, uint8_t, H1)
2714
-DO_3OP_PAIR(gvec_umaxp_h, MAX, uint16_t, H2)
2715
-DO_3OP_PAIR(gvec_umaxp_s, MAX, uint32_t, H4)
2571
+DO_3OP_PAIR_NO_STATUS(gvec_umaxp_b, MAX, uint8_t, H1)
2572
+DO_3OP_PAIR_NO_STATUS(gvec_umaxp_h, MAX, uint16_t, H2)
2573
+DO_3OP_PAIR_NO_STATUS(gvec_umaxp_s, MAX, uint32_t, H4)
2574
2717
-DO_3OP_PAIR(gvec_sminp_b, MIN, int8_t, H1)
2718
-DO_3OP_PAIR(gvec_sminp_h, MIN, int16_t, H2)
2719
-DO_3OP_PAIR(gvec_sminp_s, MIN, int32_t, H4)
2575
+DO_3OP_PAIR_NO_STATUS(gvec_sminp_b, MIN, int8_t, H1)
2576
+DO_3OP_PAIR_NO_STATUS(gvec_sminp_h, MIN, int16_t, H2)
2577
+DO_3OP_PAIR_NO_STATUS(gvec_sminp_s, MIN, int32_t, H4)
2578
2721
-DO_3OP_PAIR(gvec_uminp_b, MIN, uint8_t, H1)
2722
-DO_3OP_PAIR(gvec_uminp_h, MIN, uint16_t, H2)
2723
-DO_3OP_PAIR(gvec_uminp_s, MIN, uint32_t, H4)
2579
+DO_3OP_PAIR_NO_STATUS(gvec_uminp_b, MIN, uint8_t, H1)
2580
+DO_3OP_PAIR_NO_STATUS(gvec_uminp_h, MIN, uint16_t, H2)
2581
+DO_3OP_PAIR_NO_STATUS(gvec_uminp_s, MIN, uint32_t, H4)
2582
2725
-#undef DO_3OP_PAIR
2583
+#undef DO_3OP_PAIR_NO_STATUS
2584
2585
#define DO_VCVT_FIXED(NAME, FUNC, TYPE) \
2586
void HELPER(NAME)(void *vd, void *vn, float_status *stat, uint32_t desc) \
@@ -2797,53 +2655,6 @@ DO_VRINT_RMODE(gvec_vrint_rm_s, helper_rints, uint32_t)
2655
2656
#undef DO_VRINT_RMODE
2657
2800
-#ifdef TARGET_AARCH64
2801
-void HELPER(simd_tblx)(void *vd, void *vm, CPUARMState *env, uint32_t desc)
2802
-{
2803
- const uint8_t *indices = vm;
2804
- size_t oprsz = simd_oprsz(desc);
2805
- uint32_t rn = extract32(desc, SIMD_DATA_SHIFT, 5);
2806
- bool is_tbx = extract32(desc, SIMD_DATA_SHIFT + 5, 1);
2807
- uint32_t table_len = desc >> (SIMD_DATA_SHIFT + 6);
2808
- union {
2809
- uint8_t b[16];
2810
- uint64_t d[2];
2811
- } result;
2812
-
2813
- /*
2814
- * We must construct the final result in a temp, lest the output
2815
- * overlaps the input table. For TBL, begin with zero; for TBX,
2816
- * begin with the original register contents. Note that we always
2817
- * copy 16 bytes here to avoid an extra branch; clearing the high
2818
- * bits of the register for oprsz == 8 is handled below.
2819
- */
2820
- if (is_tbx) {
2821
- memcpy(&result, vd, 16);
2822
- } else {
2823
- memset(&result, 0, 16);
2824
- }
2825
-
2826
- for (size_t i = 0; i < oprsz; ++i) {
2827
- uint32_t index = indices[H1(i)];
2828
-
2829
- if (index < table_len) {
2830
- /*
2831
- * Convert index (a byte offset into the virtual table
2832
- * which is a series of 128-bit vectors concatenated)
2833
- * into the correct register element, bearing in mind
2834
- * that the table can wrap around from V31 to V0.
2835
- */
2836
- const uint8_t *table = (const uint8_t *)
2837
- aa64_vfp_qreg(env, (rn + (index >> 4)) % 32);
2838
- result.b[H1(i)] = table[H1(index % 16)];
2839
- }
2840
- }
2841
-
2842
- memcpy(vd, &result, 16);
2843
- clear_tail(vd, oprsz, simd_maxsz(desc));
2844
-}
2845
-#endif
2846
-
2658
/*
2659
* NxN -> N highpart multiply
2660
*
target/arm/tcg/vec_helper64.c
new
+142
@@ -0,0 +1,142 @@
1
+/*
2
+ * ARM AdvSIMD / SVE Vector Operations
3
+ *
4
+ * Copyright (c) 2026 Linaro
5
+ *
6
+ * SPDX-License-Identifier: GPL-2.0-or-later
7
+ */
8
+
9
+#include "qemu/osdep.h"
10
+#include "cpu.h"
11
+#include "helper.h"
12
+#include "helper-a64.h"
13
+#include "helper-sme.h"
14
+#include "helper-sve.h"
15
+#include "tcg/tcg-gvec-desc.h"
16
+#include "fpu/softfloat.h"
17
+#include "qemu/int128.h"
18
+#include "crypto/clmul.h"
19
+#include "vec_internal.h"
20
+
21
+DO_3OP(gvec_fdiv_h, float16_div, float16)
22
+DO_3OP(gvec_fdiv_s, float32_div, float32)
23
+DO_3OP(gvec_fdiv_d, float64_div, float64)
24
+
25
+DO_3OP(gvec_fmulx_h, helper_advsimd_mulxh, float16)
26
+DO_3OP(gvec_fmulx_s, helper_vfp_mulxs, float32)
27
+DO_3OP(gvec_fmulx_d, helper_vfp_mulxd, float64)
28
+
29
+DO_3OP(gvec_recps_h, helper_recpsf_f16, float16)
30
+DO_3OP(gvec_recps_s, helper_recpsf_f32, float32)
31
+DO_3OP(gvec_recps_d, helper_recpsf_f64, float64)
32
+
33
+DO_3OP(gvec_rsqrts_h, helper_rsqrtsf_f16, float16)
34
+DO_3OP(gvec_rsqrts_s, helper_rsqrtsf_f32, float32)
35
+DO_3OP(gvec_rsqrts_d, helper_rsqrtsf_f64, float64)
36
+
37
+DO_3OP(gvec_ah_recps_h, helper_recpsf_ah_f16, float16)
38
+DO_3OP(gvec_ah_recps_s, helper_recpsf_ah_f32, float32)
39
+DO_3OP(gvec_ah_recps_d, helper_recpsf_ah_f64, float64)
40
+
41
+DO_3OP(gvec_ah_rsqrts_h, helper_rsqrtsf_ah_f16, float16)
42
+DO_3OP(gvec_ah_rsqrts_s, helper_rsqrtsf_ah_f32, float32)
43
+DO_3OP(gvec_ah_rsqrts_d, helper_rsqrtsf_ah_f64, float64)
44
+
45
+DO_3OP(gvec_ah_fmax_h, helper_vfp_ah_maxh, float16)
46
+DO_3OP(gvec_ah_fmax_s, helper_vfp_ah_maxs, float32)
47
+DO_3OP(gvec_ah_fmax_d, helper_vfp_ah_maxd, float64)
48
+
49
+DO_3OP(gvec_ah_fmin_h, helper_vfp_ah_minh, float16)
50
+DO_3OP(gvec_ah_fmin_s, helper_vfp_ah_mins, float32)
51
+DO_3OP(gvec_ah_fmin_d, helper_vfp_ah_mind, float64)
52
+
53
+DO_3OP(gvec_fmax_b16, bfloat16_max, bfloat16)
54
+DO_3OP(gvec_fmin_b16, bfloat16_min, bfloat16)
55
+DO_3OP(gvec_fmaxnum_b16, bfloat16_maxnum, bfloat16)
56
+DO_3OP(gvec_fminnum_b16, bfloat16_minnum, bfloat16)
57
+DO_3OP(gvec_ah_fmax_b16, helper_sme2_ah_fmax_b16, bfloat16)
58
+DO_3OP(gvec_ah_fmin_b16, helper_sme2_ah_fmin_b16, bfloat16)
59
+
60
+#define nop(N, M, S) (M)
61
+
62
+DO_FMUL_IDX(gvec_fmulx_idx_h, nop, helper_advsimd_mulxh, float16, H2)
63
+DO_FMUL_IDX(gvec_fmulx_idx_s, nop, helper_vfp_mulxs, float32, H4)
64
+DO_FMUL_IDX(gvec_fmulx_idx_d, nop, helper_vfp_mulxd, float64, H8)
65
+
66
+#undef nop
67
+
68
+void HELPER(sve2_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc)
69
+{
70
+ int shift = simd_data(desc) * 8;
71
+ intptr_t i, opr_sz = simd_oprsz(desc);
72
+ uint64_t *d = vd, *n = vn, *m = vm;
73
+
74
+ for (i = 0; i < opr_sz / 8; ++i) {
75
+ d[i] = clmul_8x4_even(n[i] >> shift, m[i] >> shift);
76
+ }
77
+}
78
+
79
+void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc)
80
+{
81
+ intptr_t sel = H4(simd_data(desc));
82
+ intptr_t i, opr_sz = simd_oprsz(desc);
83
+ uint32_t *n = vn, *m = vm;
84
+ uint64_t *d = vd;
85
+
86
+ for (i = 0; i < opr_sz / 8; ++i) {
87
+ d[i] = clmul_32(n[2 * i + sel], m[2 * i + sel]);
88
+ }
89
+}
90
+
91
+DO_3OP_PAIR(gvec_ah_fmaxp_h, helper_vfp_ah_maxh, float16, H2)
92
+DO_3OP_PAIR(gvec_ah_fmaxp_s, helper_vfp_ah_maxs, float32, H4)
93
+DO_3OP_PAIR(gvec_ah_fmaxp_d, helper_vfp_ah_maxd, float64, /**/)
94
+
95
+DO_3OP_PAIR(gvec_ah_fminp_h, helper_vfp_ah_minh, float16, H2)
96
+DO_3OP_PAIR(gvec_ah_fminp_s, helper_vfp_ah_mins, float32, H4)
97
+DO_3OP_PAIR(gvec_ah_fminp_d, helper_vfp_ah_mind, float64, /**/)
98
+
99
+void HELPER(simd_tblx)(void *vd, void *vm, CPUARMState *env, uint32_t desc)
100
+{
101
+ const uint8_t *indices = vm;
102
+ size_t oprsz = simd_oprsz(desc);
103
+ uint32_t rn = extract32(desc, SIMD_DATA_SHIFT, 5);
104
+ bool is_tbx = extract32(desc, SIMD_DATA_SHIFT + 5, 1);
105
+ uint32_t table_len = desc >> (SIMD_DATA_SHIFT + 6);
106
+ union {
107
+ uint8_t b[16];
108
+ uint64_t d[2];
109
+ } result;
110
+
111
+ /*
112
+ * We must construct the final result in a temp, lest the output
113
+ * overlaps the input table. For TBL, begin with zero; for TBX,
114
+ * begin with the original register contents. Note that we always
115
+ * copy 16 bytes here to avoid an extra branch; clearing the high
116
+ * bits of the register for oprsz == 8 is handled below.
117
+ */
118
+ if (is_tbx) {
119
+ memcpy(&result, vd, 16);
120
+ } else {
121
+ memset(&result, 0, 16);
122
+ }
123
+
124
+ for (size_t i = 0; i < oprsz; ++i) {
125
+ uint32_t index = indices[H1(i)];
126
+
127
+ if (index < table_len) {
128
+ /*
129
+ * Convert index (a byte offset into the virtual table
130
+ * which is a series of 128-bit vectors concatenated)
131
+ * into the correct register element, bearing in mind
132
+ * that the table can wrap around from V31 to V0.
133
+ */
134
+ const uint8_t *table = (const uint8_t *)
135
+ aa64_vfp_qreg(env, (rn + (index >> 4)) % 32);
136
+ result.b[H1(i)] = table[H1(index % 16)];
137
+ }
138
+ }
139
+
140
+ memcpy(vd, &result, 16);
141
+ clear_tail(vd, oprsz, simd_maxsz(desc));
142
+}
target/arm/tcg/vec_internal.h
+49
@@ -450,4 +450,53 @@ static inline void depositn(uint64_t *p, unsigned pos,
450
}
451
}
452
453
+#define DO_3OP(NAME, FUNC, TYPE) \
454
+void HELPER(NAME)(void *vd, void *vn, void *vm, \
455
+ float_status * stat, uint32_t desc) \
456
+{ \
457
+ intptr_t i, oprsz = simd_oprsz(desc); \
458
+ TYPE *d = vd, *n = vn, *m = vm; \
459
+ for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
460
+ d[i] = FUNC(n[i], m[i], stat); \
461
+ } \
462
+ clear_tail(d, oprsz, simd_maxsz(desc)); \
463
+}
464
+
465
+#define DO_3OP_PAIR(NAME, FUNC, TYPE, H) \
466
+void HELPER(NAME)(void *vd, void *vn, void *vm, \
467
+ float_status * stat, uint32_t desc) \
468
+{ \
469
+ ARMVectorReg scratch; \
470
+ intptr_t oprsz = simd_oprsz(desc); \
471
+ intptr_t half = oprsz / sizeof(TYPE) / 2; \
472
+ TYPE *d = vd, *n = vn, *m = vm; \
473
+ if (unlikely(d == m)) { \
474
+ m = memcpy(&scratch, m, oprsz); \
475
+ } \
476
+ for (intptr_t i = 0; i < half; ++i) { \
477
+ d[H(i)] = FUNC(n[H(i * 2)], n[H(i * 2 + 1)], stat); \
478
+ } \
479
+ for (intptr_t i = 0; i < half; ++i) { \
480
+ d[H(i + half)] = FUNC(m[H(i * 2)], m[H(i * 2 + 1)], stat); \
481
+ } \
482
+ clear_tail(d, oprsz, simd_maxsz(desc)); \
483
+}
484
+
485
+#define DO_FMUL_IDX(NAME, ADD, MUL, TYPE, H) \
486
+void HELPER(NAME)(void *vd, void *vn, void *vm, \
487
+ float_status * stat, uint32_t desc) \
488
+{ \
489
+ intptr_t i, j, oprsz = simd_oprsz(desc); \
490
+ intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
491
+ intptr_t idx = simd_data(desc); \
492
+ TYPE *d = vd, *n = vn, *m = vm; \
493
+ for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
494
+ TYPE mm = m[H(i + idx)]; \
495
+ for (j = 0; j < segment; j++) { \
496
+ d[i + j] = ADD(d[i + j], MUL(n[i + j], mm, stat), stat); \
497
+ } \
498
+ } \
499
+ clear_tail(d, oprsz, simd_maxsz(desc)); \
500
+}
501
+
502
#endif /* TARGET_ARM_VEC_INTERNAL_H */