sve_helper.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 8,754 lines 321 KB

Raw

1	/*
2	* ARM SVE Operations
3	*
4	* Copyright (c) 2018 Linaro, Ltd.
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#include "qemu/osdep.h"
21	#include "cpu.h"
22	#include "internals.h"
23	#include "exec/page-protection.h"
24	#include "exec/target_page.h"
25	#include "exec/tlb-flags.h"
26	#include "helper.h"
27	#include "helper-a64.h"
28	#include "helper-sve.h"
29	#include "tcg/tcg-gvec-desc.h"
30	#include "fpu/softfloat.h"
31	#include "tcg/tcg.h"
32	#include "vec_internal.h"
33	#include "sve_ldst_internal.h"
34	#include "accel/tcg/cpu-ldst.h"
35	#include "accel/tcg/helper-retaddr.h"
36	#include "accel/tcg/cpu-ops.h"
37	#include "accel/tcg/probe.h"
38	#ifdef CONFIG_USER_ONLY
39	#include "user/page-protection.h"
40	#endif
41
42	#define HELPER_H "tcg/helper-sve-defs.h"
43	#include "exec/helper-info.c.inc"
44
45	/* Return a value for NZCV as per the ARM PredTest pseudofunction.
46	*
47	* The return value has bit 31 set if N is set, bit 1 set if Z is clear,
48	* and bit 0 set if C is set. Compare the definitions of these variables
49	* within CPUARMState.
50	*/
51
52	/* For no G bits set, NZCV = C. */
53	#define PREDTEST_INIT 1
54
55	/* This is an iterative function, called for each Pd and Pg word
56	* moving forward.
57	*/
58	static uint32_t iter_predtest_fwd(uint64_t d, uint64_t g, uint32_t flags)
59	{
60	if (likely(g)) {
61	/* Compute N from first D & G.
62	Use bit 2 to signal first G bit seen. */
63	if (!(flags & 4)) {
64	flags \|= ((d & (g & -g)) != 0) << 31;
65	flags \|= 4;
66	}
67
68	/* Accumulate Z from each D & G. */
69	flags \|= ((d & g) != 0) << 1;
70
71	/* Compute C from last !(D & G). Replace previous. */
72	flags = deposit32(flags, 0, 1, (d & pow2floor(g)) == 0);
73	}
74	return flags;
75	}
76
77	/* This is an iterative function, called for each Pd and Pg word
78	* moving backward.
79	*/
80	static uint32_t iter_predtest_bwd(uint64_t d, uint64_t g, uint32_t flags)
81	{
82	if (likely(g)) {
83	/* Compute C from first (i.e last) !(D & G).
84	Use bit 2 to signal first G bit seen. */
85	if (!(flags & 4)) {
86	flags += 4 - 1; /* add bit 2, subtract C from PREDTEST_INIT */
87	flags \|= (d & pow2floor(g)) == 0;
88	}
89
90	/* Accumulate Z from each D & G. */
91	flags \|= ((d & g) != 0) << 1;
92
93	/* Compute N from last (i.e first) D & G. Replace previous. */
94	flags = deposit32(flags, 31, 1, (d & (g & -g)) != 0);
95	}
96	return flags;
97	}
98
99	/* The same for a single word predicate. */
100	uint32_t HELPER(sve_predtest1)(uint64_t d, uint64_t g)
101	{
102	return iter_predtest_fwd(d, g, PREDTEST_INIT);
103	}
104
105	/* The same for a multi-word predicate. */
106	uint32_t HELPER(sve_predtest)(void vd, void vg, uint32_t words)
107	{
108	uint32_t flags = PREDTEST_INIT;
109	uint64_t d = vd, g = vg;
110	uintptr_t i = 0;
111
112	do {
113	flags = iter_predtest_fwd(d[i], g[i], flags);
114	} while (++i < words);
115
116	return flags;
117	}
118
119	/* Similarly for single word elements. */
120	static inline uint64_t expand_pred_s(uint8_t byte)
121	{
122	static const uint64_t word[] = {
123	[0x01] = 0x00000000ffffffffull,
124	[0x10] = 0xffffffff00000000ull,
125	[0x11] = 0xffffffffffffffffull,
126	};
127	return word[byte & 0x11];
128	}
129
130	static inline uint64_t expand_pred_d(uint8_t byte)
131	{
132	return -(uint64_t)(byte & 1);
133	}
134
135	#define LOGICAL_PPPP(NAME, FUNC) \
136	void HELPER(NAME)(void vd, void vn, void vm, void vg, uint32_t desc) \
137	{ \
138	uintptr_t opr_sz = simd_oprsz(desc); \
139	uint64_t d = vd, n = vn, m = vm, g = vg; \
140	uintptr_t i; \
141	for (i = 0; i < opr_sz / 8; ++i) { \
142	d[i] = FUNC(n[i], m[i], g[i]); \
143	} \
144	}
145
146	#define DO_AND(N, M, G) (((N) & (M)) & (G))
147	#define DO_BIC(N, M, G) (((N) & ~(M)) & (G))
148	#define DO_EOR(N, M, G) (((N) ^ (M)) & (G))
149	#define DO_ORR(N, M, G) (((N) \| (M)) & (G))
150	#define DO_ORN(N, M, G) (((N) \| ~(M)) & (G))
151	#define DO_NOR(N, M, G) (~((N) \| (M)) & (G))
152	#define DO_NAND(N, M, G) (~((N) & (M)) & (G))
153	#define DO_SEL(N, M, G) (((N) & (G)) \| ((M) & ~(G)))
154
155	LOGICAL_PPPP(sve_and_pppp, DO_AND)
156	LOGICAL_PPPP(sve_bic_pppp, DO_BIC)
157	LOGICAL_PPPP(sve_eor_pppp, DO_EOR)
158	LOGICAL_PPPP(sve_sel_pppp, DO_SEL)
159	LOGICAL_PPPP(sve_orr_pppp, DO_ORR)
160	LOGICAL_PPPP(sve_orn_pppp, DO_ORN)
161	LOGICAL_PPPP(sve_nor_pppp, DO_NOR)
162	LOGICAL_PPPP(sve_nand_pppp, DO_NAND)
163
164	#undef DO_AND
165	#undef DO_BIC
166	#undef DO_EOR
167	#undef DO_ORR
168	#undef DO_ORN
169	#undef DO_NOR
170	#undef DO_NAND
171	#undef DO_SEL
172	#undef LOGICAL_PPPP
173
174	/* Fully general three-operand expander, controlled by a predicate.
175	* This is complicated by the host-endian storage of the register file.
176	*/
177	/* ??? I don't expect the compiler could ever vectorize this itself.
178	* With some tables we can convert bit masks to byte masks, and with
179	* extra care wrt byte/word ordering we could use gcc generic vectors
180	* and do 16 bytes at a time.
181	*/
182	#define DO_ZPZZ(NAME, TYPE, H, OP) \
183	void HELPER(NAME)(void vd, void vn, void vm, void vg, uint32_t desc) \
184	{ \
185	intptr_t i, opr_sz = simd_oprsz(desc); \
186	for (i = 0; i < opr_sz; ) { \
187	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
188	do { \
189	if (pg & 1) { \
190	TYPE nn = (TYPE )(vn + H(i)); \
191	TYPE mm = (TYPE )(vm + H(i)); \
192	(TYPE )(vd + H(i)) = OP(nn, mm); \
193	} \
194	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
195	} while (i & 15); \
196	} \
197	}
198
199	/* Similarly, specialized for 64-bit operands. */
200	#define DO_ZPZZ_D(NAME, TYPE, OP) \
201	void HELPER(NAME)(void vd, void vn, void vm, void vg, uint32_t desc) \
202	{ \
203	intptr_t i, opr_sz = simd_oprsz(desc) / 8; \
204	TYPE d = vd, n = vn, *m = vm; \
205	uint8_t *pg = vg; \
206	for (i = 0; i < opr_sz; i += 1) { \
207	if (pg[H1(i)] & 1) { \
208	TYPE nn = n[i], mm = m[i]; \
209	d[i] = OP(nn, mm); \
210	} \
211	} \
212	}
213
214	#define DO_AND(N, M) (N & M)
215	#define DO_EOR(N, M) (N ^ M)
216	#define DO_ORR(N, M) (N \| M)
217	#define DO_BIC(N, M) (N & ~M)
218	#define DO_ORC(N, M) (N \| ~M)
219	#define DO_ADD(N, M) (N + M)
220	#define DO_SUB(N, M) (N - M)
221	#define DO_MAX(N, M) ((N) >= (M) ? (N) : (M))
222	#define DO_MIN(N, M) ((N) >= (M) ? (M) : (N))
223	#define DO_ABD(N, M) ((N) >= (M) ? (N) - (M) : (M) - (N))
224	#define DO_MUL(N, M) (N * M)
225
226
227	/*
228	* We must avoid the C undefined behaviour cases: division by
229	* zero and signed division of INT_MIN by -1. Both of these
230	* have architecturally defined required results for Arm.
231	* We special case all signed divisions by -1 to avoid having
232	* to deduce the minimum integer for the type involved.
233	*/
234	#define DO_SDIV(N, M) (unlikely(M == 0) ? 0 : unlikely(M == -1) ? -N : N / M)
235	#define DO_UDIV(N, M) (unlikely(M == 0) ? 0 : N / M)
236
237	DO_ZPZZ(sve_and_zpzz_b, uint8_t, H1, DO_AND)
238	DO_ZPZZ(sve_and_zpzz_h, uint16_t, H1_2, DO_AND)
239	DO_ZPZZ(sve_and_zpzz_s, uint32_t, H1_4, DO_AND)
240	DO_ZPZZ_D(sve_and_zpzz_d, uint64_t, DO_AND)
241
242	DO_ZPZZ(sve_orr_zpzz_b, uint8_t, H1, DO_ORR)
243	DO_ZPZZ(sve_orr_zpzz_h, uint16_t, H1_2, DO_ORR)
244	DO_ZPZZ(sve_orr_zpzz_s, uint32_t, H1_4, DO_ORR)
245	DO_ZPZZ_D(sve_orr_zpzz_d, uint64_t, DO_ORR)
246
247	DO_ZPZZ(sve_eor_zpzz_b, uint8_t, H1, DO_EOR)
248	DO_ZPZZ(sve_eor_zpzz_h, uint16_t, H1_2, DO_EOR)
249	DO_ZPZZ(sve_eor_zpzz_s, uint32_t, H1_4, DO_EOR)
250	DO_ZPZZ_D(sve_eor_zpzz_d, uint64_t, DO_EOR)
251
252	DO_ZPZZ(sve_bic_zpzz_b, uint8_t, H1, DO_BIC)
253	DO_ZPZZ(sve_bic_zpzz_h, uint16_t, H1_2, DO_BIC)
254	DO_ZPZZ(sve_bic_zpzz_s, uint32_t, H1_4, DO_BIC)
255	DO_ZPZZ_D(sve_bic_zpzz_d, uint64_t, DO_BIC)
256
257	DO_ZPZZ(sve_add_zpzz_b, uint8_t, H1, DO_ADD)
258	DO_ZPZZ(sve_add_zpzz_h, uint16_t, H1_2, DO_ADD)
259	DO_ZPZZ(sve_add_zpzz_s, uint32_t, H1_4, DO_ADD)
260	DO_ZPZZ_D(sve_add_zpzz_d, uint64_t, DO_ADD)
261
262	DO_ZPZZ(sve_sub_zpzz_b, uint8_t, H1, DO_SUB)
263	DO_ZPZZ(sve_sub_zpzz_h, uint16_t, H1_2, DO_SUB)
264	DO_ZPZZ(sve_sub_zpzz_s, uint32_t, H1_4, DO_SUB)
265	DO_ZPZZ_D(sve_sub_zpzz_d, uint64_t, DO_SUB)
266
267	DO_ZPZZ(sve_smax_zpzz_b, int8_t, H1, DO_MAX)
268	DO_ZPZZ(sve_smax_zpzz_h, int16_t, H1_2, DO_MAX)
269	DO_ZPZZ(sve_smax_zpzz_s, int32_t, H1_4, DO_MAX)
270	DO_ZPZZ_D(sve_smax_zpzz_d, int64_t, DO_MAX)
271
272	DO_ZPZZ(sve_umax_zpzz_b, uint8_t, H1, DO_MAX)
273	DO_ZPZZ(sve_umax_zpzz_h, uint16_t, H1_2, DO_MAX)
274	DO_ZPZZ(sve_umax_zpzz_s, uint32_t, H1_4, DO_MAX)
275	DO_ZPZZ_D(sve_umax_zpzz_d, uint64_t, DO_MAX)
276
277	DO_ZPZZ(sve_smin_zpzz_b, int8_t, H1, DO_MIN)
278	DO_ZPZZ(sve_smin_zpzz_h, int16_t, H1_2, DO_MIN)
279	DO_ZPZZ(sve_smin_zpzz_s, int32_t, H1_4, DO_MIN)
280	DO_ZPZZ_D(sve_smin_zpzz_d, int64_t, DO_MIN)
281
282	DO_ZPZZ(sve_umin_zpzz_b, uint8_t, H1, DO_MIN)
283	DO_ZPZZ(sve_umin_zpzz_h, uint16_t, H1_2, DO_MIN)
284	DO_ZPZZ(sve_umin_zpzz_s, uint32_t, H1_4, DO_MIN)
285	DO_ZPZZ_D(sve_umin_zpzz_d, uint64_t, DO_MIN)
286
287	DO_ZPZZ(sve_sabd_zpzz_b, int8_t, H1, DO_ABD)
288	DO_ZPZZ(sve_sabd_zpzz_h, int16_t, H1_2, DO_ABD)
289	DO_ZPZZ(sve_sabd_zpzz_s, int32_t, H1_4, DO_ABD)
290	DO_ZPZZ_D(sve_sabd_zpzz_d, int64_t, DO_ABD)
291
292	DO_ZPZZ(sve_uabd_zpzz_b, uint8_t, H1, DO_ABD)
293	DO_ZPZZ(sve_uabd_zpzz_h, uint16_t, H1_2, DO_ABD)
294	DO_ZPZZ(sve_uabd_zpzz_s, uint32_t, H1_4, DO_ABD)
295	DO_ZPZZ_D(sve_uabd_zpzz_d, uint64_t, DO_ABD)
296
297	/* Because the computation type is at least twice as large as required,
298	these work for both signed and unsigned source types. */
299	static inline uint8_t do_mulh_b(int32_t n, int32_t m)
300	{
301	return (n * m) >> 8;
302	}
303
304	static inline uint16_t do_mulh_h(int32_t n, int32_t m)
305	{
306	return (n * m) >> 16;
307	}
308
309	static inline uint32_t do_mulh_s(int64_t n, int64_t m)
310	{
311	return (n * m) >> 32;
312	}
313
314	static inline uint64_t do_smulh_d(uint64_t n, uint64_t m)
315	{
316	uint64_t lo, hi;
317	muls64(&lo, &hi, n, m);
318	return hi;
319	}
320
321	static inline uint64_t do_umulh_d(uint64_t n, uint64_t m)
322	{
323	uint64_t lo, hi;
324	mulu64(&lo, &hi, n, m);
325	return hi;
326	}
327
328	DO_ZPZZ(sve_mul_zpzz_b, uint8_t, H1, DO_MUL)
329	DO_ZPZZ(sve_mul_zpzz_h, uint16_t, H1_2, DO_MUL)
330	DO_ZPZZ(sve_mul_zpzz_s, uint32_t, H1_4, DO_MUL)
331	DO_ZPZZ_D(sve_mul_zpzz_d, uint64_t, DO_MUL)
332
333	DO_ZPZZ(sve_smulh_zpzz_b, int8_t, H1, do_mulh_b)
334	DO_ZPZZ(sve_smulh_zpzz_h, int16_t, H1_2, do_mulh_h)
335	DO_ZPZZ(sve_smulh_zpzz_s, int32_t, H1_4, do_mulh_s)
336	DO_ZPZZ_D(sve_smulh_zpzz_d, uint64_t, do_smulh_d)
337
338	DO_ZPZZ(sve_umulh_zpzz_b, uint8_t, H1, do_mulh_b)
339	DO_ZPZZ(sve_umulh_zpzz_h, uint16_t, H1_2, do_mulh_h)
340	DO_ZPZZ(sve_umulh_zpzz_s, uint32_t, H1_4, do_mulh_s)
341	DO_ZPZZ_D(sve_umulh_zpzz_d, uint64_t, do_umulh_d)
342
343	DO_ZPZZ(sve_sdiv_zpzz_s, int32_t, H1_4, DO_SDIV)
344	DO_ZPZZ_D(sve_sdiv_zpzz_d, int64_t, DO_SDIV)
345
346	DO_ZPZZ(sve_udiv_zpzz_s, uint32_t, H1_4, DO_UDIV)
347	DO_ZPZZ_D(sve_udiv_zpzz_d, uint64_t, DO_UDIV)
348
349	/* Note that all bits of the shift are significant
350	and not modulo the element size. */
351	#define DO_ASR(N, M) (N >> MIN(M, sizeof(N) * 8 - 1))
352	#define DO_LSR(N, M) (M < sizeof(N) * 8 ? N >> M : 0)
353	#define DO_LSL(N, M) (M < sizeof(N) * 8 ? N << M : 0)
354
355	DO_ZPZZ(sve_asr_zpzz_b, int8_t, H1, DO_ASR)
356	DO_ZPZZ(sve_lsr_zpzz_b, uint8_t, H1_2, DO_LSR)
357	DO_ZPZZ(sve_lsl_zpzz_b, uint8_t, H1_4, DO_LSL)
358
359	DO_ZPZZ(sve_asr_zpzz_h, int16_t, H1, DO_ASR)
360	DO_ZPZZ(sve_lsr_zpzz_h, uint16_t, H1_2, DO_LSR)
361	DO_ZPZZ(sve_lsl_zpzz_h, uint16_t, H1_4, DO_LSL)
362
363	DO_ZPZZ(sve_asr_zpzz_s, int32_t, H1, DO_ASR)
364	DO_ZPZZ(sve_lsr_zpzz_s, uint32_t, H1_2, DO_LSR)
365	DO_ZPZZ(sve_lsl_zpzz_s, uint32_t, H1_4, DO_LSL)
366
367	DO_ZPZZ_D(sve_asr_zpzz_d, int64_t, DO_ASR)
368	DO_ZPZZ_D(sve_lsr_zpzz_d, uint64_t, DO_LSR)
369	DO_ZPZZ_D(sve_lsl_zpzz_d, uint64_t, DO_LSL)
370
371	static inline uint16_t do_sadalp_h(int16_t n, int16_t m)
372	{
373	int8_t n1 = n, n2 = n >> 8;
374	return m + n1 + n2;
375	}
376
377	static inline uint32_t do_sadalp_s(int32_t n, int32_t m)
378	{
379	int16_t n1 = n, n2 = n >> 16;
380	return m + n1 + n2;
381	}
382
383	static inline uint64_t do_sadalp_d(int64_t n, int64_t m)
384	{
385	int32_t n1 = n, n2 = n >> 32;
386	return m + n1 + n2;
387	}
388
389	DO_ZPZZ(sve2_sadalp_zpzz_h, int16_t, H1_2, do_sadalp_h)
390	DO_ZPZZ(sve2_sadalp_zpzz_s, int32_t, H1_4, do_sadalp_s)
391	DO_ZPZZ_D(sve2_sadalp_zpzz_d, int64_t, do_sadalp_d)
392
393	static inline uint16_t do_uadalp_h(uint16_t n, uint16_t m)
394	{
395	uint8_t n1 = n, n2 = n >> 8;
396	return m + n1 + n2;
397	}
398
399	static inline uint32_t do_uadalp_s(uint32_t n, uint32_t m)
400	{
401	uint16_t n1 = n, n2 = n >> 16;
402	return m + n1 + n2;
403	}
404
405	static inline uint64_t do_uadalp_d(uint64_t n, uint64_t m)
406	{
407	uint32_t n1 = n, n2 = n >> 32;
408	return m + n1 + n2;
409	}
410
411	DO_ZPZZ(sve2_uadalp_zpzz_h, uint16_t, H1_2, do_uadalp_h)
412	DO_ZPZZ(sve2_uadalp_zpzz_s, uint32_t, H1_4, do_uadalp_s)
413	DO_ZPZZ_D(sve2_uadalp_zpzz_d, uint64_t, do_uadalp_d)
414
415	#define do_srshl_b(n, m) do_sqrshl_bhs(n, m, 8, true, NULL)
416	#define do_srshl_h(n, m) do_sqrshl_bhs(n, m, 16, true, NULL)
417	#define do_srshl_s(n, m) do_sqrshl_bhs(n, m, 32, true, NULL)
418	#define do_srshl_d(n, m) do_sqrshl_d(n, m, true, NULL)
419
420	DO_ZPZZ(sve2_srshl_zpzz_b, int8_t, H1, do_srshl_b)
421	DO_ZPZZ(sve2_srshl_zpzz_h, int16_t, H1_2, do_srshl_h)
422	DO_ZPZZ(sve2_srshl_zpzz_s, int32_t, H1_4, do_srshl_s)
423	DO_ZPZZ_D(sve2_srshl_zpzz_d, int64_t, do_srshl_d)
424
425	#define do_urshl_b(n, m) do_uqrshl_bhs(n, (int8_t)m, 8, true, NULL)
426	#define do_urshl_h(n, m) do_uqrshl_bhs(n, (int16_t)m, 16, true, NULL)
427	#define do_urshl_s(n, m) do_uqrshl_bhs(n, m, 32, true, NULL)
428	#define do_urshl_d(n, m) do_uqrshl_d(n, m, true, NULL)
429
430	DO_ZPZZ(sve2_urshl_zpzz_b, uint8_t, H1, do_urshl_b)
431	DO_ZPZZ(sve2_urshl_zpzz_h, uint16_t, H1_2, do_urshl_h)
432	DO_ZPZZ(sve2_urshl_zpzz_s, uint32_t, H1_4, do_urshl_s)
433	DO_ZPZZ_D(sve2_urshl_zpzz_d, uint64_t, do_urshl_d)
434
435	/*
436	* Unlike the NEON and AdvSIMD versions, there is no QC bit to set.
437	* We pass in a pointer to a dummy saturation field to trigger
438	* the saturating arithmetic but discard the information about
439	* whether it has occurred.
440	*/
441	#define do_sqshl_b(n, m) \
442	({ uint32_t discard; do_sqrshl_bhs(n, m, 8, false, &discard); })
443	#define do_sqshl_h(n, m) \
444	({ uint32_t discard; do_sqrshl_bhs(n, m, 16, false, &discard); })
445	#define do_sqshl_s(n, m) \
446	({ uint32_t discard; do_sqrshl_bhs(n, m, 32, false, &discard); })
447	#define do_sqshl_d(n, m) \
448	({ uint32_t discard; do_sqrshl_d(n, m, false, &discard); })
449
450	DO_ZPZZ(sve2_sqshl_zpzz_b, int8_t, H1_2, do_sqshl_b)
451	DO_ZPZZ(sve2_sqshl_zpzz_h, int16_t, H1_2, do_sqshl_h)
452	DO_ZPZZ(sve2_sqshl_zpzz_s, int32_t, H1_4, do_sqshl_s)
453	DO_ZPZZ_D(sve2_sqshl_zpzz_d, int64_t, do_sqshl_d)
454
455	#define do_uqshl_b(n, m) \
456	({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, false, &discard); })
457	#define do_uqshl_h(n, m) \
458	({ uint32_t discard; do_uqrshl_bhs(n, (int16_t)m, 16, false, &discard); })
459	#define do_uqshl_s(n, m) \
460	({ uint32_t discard; do_uqrshl_bhs(n, m, 32, false, &discard); })
461	#define do_uqshl_d(n, m) \
462	({ uint32_t discard; do_uqrshl_d(n, m, false, &discard); })
463
464	DO_ZPZZ(sve2_uqshl_zpzz_b, uint8_t, H1_2, do_uqshl_b)
465	DO_ZPZZ(sve2_uqshl_zpzz_h, uint16_t, H1_2, do_uqshl_h)
466	DO_ZPZZ(sve2_uqshl_zpzz_s, uint32_t, H1_4, do_uqshl_s)
467	DO_ZPZZ_D(sve2_uqshl_zpzz_d, uint64_t, do_uqshl_d)
468
469	#define do_sqrshl_b(n, m) \
470	({ uint32_t discard; do_sqrshl_bhs(n, m, 8, true, &discard); })
471	#define do_sqrshl_h(n, m) \
472	({ uint32_t discard; do_sqrshl_bhs(n, m, 16, true, &discard); })
473	#define do_sqrshl_s(n, m) \
474	({ uint32_t discard; do_sqrshl_bhs(n, m, 32, true, &discard); })
475	#define do_sqrshl_d(n, m) \
476	({ uint32_t discard; do_sqrshl_d(n, m, true, &discard); })
477
478	DO_ZPZZ(sve2_sqrshl_zpzz_b, int8_t, H1_2, do_sqrshl_b)
479	DO_ZPZZ(sve2_sqrshl_zpzz_h, int16_t, H1_2, do_sqrshl_h)
480	DO_ZPZZ(sve2_sqrshl_zpzz_s, int32_t, H1_4, do_sqrshl_s)
481	DO_ZPZZ_D(sve2_sqrshl_zpzz_d, int64_t, do_sqrshl_d)
482
483	#undef do_sqrshl_d
484
485	#define do_uqrshl_b(n, m) \
486	({ uint32_t discard; do_uqrshl_bhs(n, (int8_t)m, 8, true, &discard); })
487	#define do_uqrshl_h(n, m) \
488	({ uint32_t discard; do_uqrshl_bhs(n, (int16_t)m, 16, true, &discard); })
489	#define do_uqrshl_s(n, m) \
490	({ uint32_t discard; do_uqrshl_bhs(n, m, 32, true, &discard); })
491	#define do_uqrshl_d(n, m) \
492	({ uint32_t discard; do_uqrshl_d(n, m, true, &discard); })
493
494	DO_ZPZZ(sve2_uqrshl_zpzz_b, uint8_t, H1_2, do_uqrshl_b)
495	DO_ZPZZ(sve2_uqrshl_zpzz_h, uint16_t, H1_2, do_uqrshl_h)
496	DO_ZPZZ(sve2_uqrshl_zpzz_s, uint32_t, H1_4, do_uqrshl_s)
497	DO_ZPZZ_D(sve2_uqrshl_zpzz_d, uint64_t, do_uqrshl_d)
498
499	#undef do_uqrshl_d
500
501	#define DO_HADD_BHS(n, m) (((int64_t)n + m) >> 1)
502	#define DO_HADD_D(n, m) ((n >> 1) + (m >> 1) + (n & m & 1))
503
504	DO_ZPZZ(sve2_shadd_zpzz_b, int8_t, H1, DO_HADD_BHS)
505	DO_ZPZZ(sve2_shadd_zpzz_h, int16_t, H1_2, DO_HADD_BHS)
506	DO_ZPZZ(sve2_shadd_zpzz_s, int32_t, H1_4, DO_HADD_BHS)
507	DO_ZPZZ_D(sve2_shadd_zpzz_d, int64_t, DO_HADD_D)
508
509	DO_ZPZZ(sve2_uhadd_zpzz_b, uint8_t, H1, DO_HADD_BHS)
510	DO_ZPZZ(sve2_uhadd_zpzz_h, uint16_t, H1_2, DO_HADD_BHS)
511	DO_ZPZZ(sve2_uhadd_zpzz_s, uint32_t, H1_4, DO_HADD_BHS)
512	DO_ZPZZ_D(sve2_uhadd_zpzz_d, uint64_t, DO_HADD_D)
513
514	#define DO_RHADD_BHS(n, m) (((int64_t)n + m + 1) >> 1)
515	#define DO_RHADD_D(n, m) ((n >> 1) + (m >> 1) + ((n \| m) & 1))
516
517	DO_ZPZZ(sve2_srhadd_zpzz_b, int8_t, H1, DO_RHADD_BHS)
518	DO_ZPZZ(sve2_srhadd_zpzz_h, int16_t, H1_2, DO_RHADD_BHS)
519	DO_ZPZZ(sve2_srhadd_zpzz_s, int32_t, H1_4, DO_RHADD_BHS)
520	DO_ZPZZ_D(sve2_srhadd_zpzz_d, int64_t, DO_RHADD_D)
521
522	DO_ZPZZ(sve2_urhadd_zpzz_b, uint8_t, H1, DO_RHADD_BHS)
523	DO_ZPZZ(sve2_urhadd_zpzz_h, uint16_t, H1_2, DO_RHADD_BHS)
524	DO_ZPZZ(sve2_urhadd_zpzz_s, uint32_t, H1_4, DO_RHADD_BHS)
525	DO_ZPZZ_D(sve2_urhadd_zpzz_d, uint64_t, DO_RHADD_D)
526
527	#define DO_HSUB_BHS(n, m) (((int64_t)n - m) >> 1)
528	#define DO_HSUB_D(n, m) ((n >> 1) - (m >> 1) - (~n & m & 1))
529
530	DO_ZPZZ(sve2_shsub_zpzz_b, int8_t, H1, DO_HSUB_BHS)
531	DO_ZPZZ(sve2_shsub_zpzz_h, int16_t, H1_2, DO_HSUB_BHS)
532	DO_ZPZZ(sve2_shsub_zpzz_s, int32_t, H1_4, DO_HSUB_BHS)
533	DO_ZPZZ_D(sve2_shsub_zpzz_d, int64_t, DO_HSUB_D)
534
535	DO_ZPZZ(sve2_uhsub_zpzz_b, uint8_t, H1, DO_HSUB_BHS)
536	DO_ZPZZ(sve2_uhsub_zpzz_h, uint16_t, H1_2, DO_HSUB_BHS)
537	DO_ZPZZ(sve2_uhsub_zpzz_s, uint32_t, H1_4, DO_HSUB_BHS)
538	DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D)
539
540	#define DO_SQADD_B(n, m) do_ssat_b((int64_t)n + m)
541	#define DO_SQADD_H(n, m) do_ssat_h((int64_t)n + m)
542	#define DO_SQADD_S(n, m) do_ssat_s((int64_t)n + m)
543
544	static inline int64_t do_sqadd_d(int64_t n, int64_t m)
545	{
546	int64_t r = n + m;
547	if (((r ^ n) & ~(n ^ m)) < 0) {
548	/* Signed overflow. */
549	return r < 0 ? INT64_MAX : INT64_MIN;
550	}
551	return r;
552	}
553
554	DO_ZPZZ(sve2_sqadd_zpzz_b, int8_t, H1, DO_SQADD_B)
555	DO_ZPZZ(sve2_sqadd_zpzz_h, int16_t, H1_2, DO_SQADD_H)
556	DO_ZPZZ(sve2_sqadd_zpzz_s, int32_t, H1_4, DO_SQADD_S)
557	DO_ZPZZ_D(sve2_sqadd_zpzz_d, int64_t, do_sqadd_d)
558
559	#define DO_UQADD_B(n, m) do_usat_b((int64_t)n + m)
560	#define DO_UQADD_H(n, m) do_usat_h((int64_t)n + m)
561	#define DO_UQADD_S(n, m) do_usat_s((int64_t)n + m)
562
563	static inline uint64_t do_uqadd_d(uint64_t n, uint64_t m)
564	{
565	uint64_t r = n + m;
566	return r < n ? UINT64_MAX : r;
567	}
568
569	DO_ZPZZ(sve2_uqadd_zpzz_b, uint8_t, H1, DO_UQADD_B)
570	DO_ZPZZ(sve2_uqadd_zpzz_h, uint16_t, H1_2, DO_UQADD_H)
571	DO_ZPZZ(sve2_uqadd_zpzz_s, uint32_t, H1_4, DO_UQADD_S)
572	DO_ZPZZ_D(sve2_uqadd_zpzz_d, uint64_t, do_uqadd_d)
573
574	#define DO_SQSUB_B(n, m) do_ssat_b((int64_t)n - m)
575	#define DO_SQSUB_H(n, m) do_ssat_h((int64_t)n - m)
576	#define DO_SQSUB_S(n, m) do_ssat_s((int64_t)n - m)
577
578	static inline int64_t do_sqsub_d(int64_t n, int64_t m)
579	{
580	int64_t r = n - m;
581	if (((r ^ n) & (n ^ m)) < 0) {
582	/* Signed overflow. */
583	return r < 0 ? INT64_MAX : INT64_MIN;
584	}
585	return r;
586	}
587
588	DO_ZPZZ(sve2_sqsub_zpzz_b, int8_t, H1, DO_SQSUB_B)
589	DO_ZPZZ(sve2_sqsub_zpzz_h, int16_t, H1_2, DO_SQSUB_H)
590	DO_ZPZZ(sve2_sqsub_zpzz_s, int32_t, H1_4, DO_SQSUB_S)
591	DO_ZPZZ_D(sve2_sqsub_zpzz_d, int64_t, do_sqsub_d)
592
593	#define DO_UQSUB_B(n, m) do_usat_b((int64_t)n - m)
594	#define DO_UQSUB_H(n, m) do_usat_h((int64_t)n - m)
595	#define DO_UQSUB_S(n, m) do_usat_s((int64_t)n - m)
596
597	static inline uint64_t do_uqsub_d(uint64_t n, uint64_t m)
598	{
599	return n > m ? n - m : 0;
600	}
601
602	DO_ZPZZ(sve2_uqsub_zpzz_b, uint8_t, H1, DO_UQSUB_B)
603	DO_ZPZZ(sve2_uqsub_zpzz_h, uint16_t, H1_2, DO_UQSUB_H)
604	DO_ZPZZ(sve2_uqsub_zpzz_s, uint32_t, H1_4, DO_UQSUB_S)
605	DO_ZPZZ_D(sve2_uqsub_zpzz_d, uint64_t, do_uqsub_d)
606
607	#define DO_SUQADD_B(n, m) do_ssat_b((int64_t)(int8_t)n + m)
608	#define DO_SUQADD_H(n, m) do_ssat_h((int64_t)(int16_t)n + m)
609	#define DO_SUQADD_S(n, m) do_ssat_s((int64_t)(int32_t)n + m)
610
611	static inline int64_t do_suqadd_d(int64_t n, uint64_t m)
612	{
613	uint64_t r = n + m;
614
615	if (n < 0) {
616	/* Note that m - abs(n) cannot underflow. */
617	if (r > INT64_MAX) {
618	/* Result is either very large positive or negative. */
619	if (m > -n) {
620	/* m > abs(n), so r is a very large positive. */
621	return INT64_MAX;
622	}
623	/* Result is negative. */
624	}
625	} else {
626	/* Both inputs are positive: check for overflow. */
627	if (r < m \|\| r > INT64_MAX) {
628	return INT64_MAX;
629	}
630	}
631	return r;
632	}
633
634	DO_ZPZZ(sve2_suqadd_zpzz_b, uint8_t, H1, DO_SUQADD_B)
635	DO_ZPZZ(sve2_suqadd_zpzz_h, uint16_t, H1_2, DO_SUQADD_H)
636	DO_ZPZZ(sve2_suqadd_zpzz_s, uint32_t, H1_4, DO_SUQADD_S)
637	DO_ZPZZ_D(sve2_suqadd_zpzz_d, uint64_t, do_suqadd_d)
638
639	#define DO_USQADD_B(n, m) do_usat_b((int64_t)n + (int8_t)m)
640	#define DO_USQADD_H(n, m) do_usat_h((int64_t)n + (int16_t)m)
641	#define DO_USQADD_S(n, m) do_usat_s((int64_t)n + (int32_t)m)
642
643	static inline uint64_t do_usqadd_d(uint64_t n, int64_t m)
644	{
645	uint64_t r = n + m;
646
647	if (m < 0) {
648	return n < -m ? 0 : r;
649	}
650	return r < n ? UINT64_MAX : r;
651	}
652
653	DO_ZPZZ(sve2_usqadd_zpzz_b, uint8_t, H1, DO_USQADD_B)
654	DO_ZPZZ(sve2_usqadd_zpzz_h, uint16_t, H1_2, DO_USQADD_H)
655	DO_ZPZZ(sve2_usqadd_zpzz_s, uint32_t, H1_4, DO_USQADD_S)
656	DO_ZPZZ_D(sve2_usqadd_zpzz_d, uint64_t, do_usqadd_d)
657
658	#undef DO_ZPZZ
659	#undef DO_ZPZZ_D
660
661	/*
662	* Three operand expander, operating on element pairs.
663	* If the slot I is even, the elements from from VN {I, I+1}.
664	* If the slot I is odd, the elements from from VM {I-1, I}.
665	* Load all of the input elements in each pair before overwriting output.
666	*/
667	#define DO_ZPZZ_PAIR(NAME, TYPE, H, OP) \
668	void HELPER(NAME)(void vd, void vn, void vm, void vg, uint32_t desc) \
669	{ \
670	intptr_t i, opr_sz = simd_oprsz(desc); \
671	for (i = 0; i < opr_sz; ) { \
672	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
673	do { \
674	TYPE n0 = (TYPE )(vn + H(i)); \
675	TYPE m0 = (TYPE )(vm + H(i)); \
676	TYPE n1 = (TYPE )(vn + H(i + sizeof(TYPE))); \
677	TYPE m1 = (TYPE )(vm + H(i + sizeof(TYPE))); \
678	if (pg & 1) { \
679	(TYPE )(vd + H(i)) = OP(n0, n1); \
680	} \
681	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
682	if (pg & 1) { \
683	(TYPE )(vd + H(i)) = OP(m0, m1); \
684	} \
685	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
686	} while (i & 15); \
687	} \
688	}
689
690	/* Similarly, specialized for 64-bit operands. */
691	#define DO_ZPZZ_PAIR_D(NAME, TYPE, OP) \
692	void HELPER(NAME)(void vd, void vn, void vm, void vg, uint32_t desc) \
693	{ \
694	intptr_t i, opr_sz = simd_oprsz(desc) / 8; \
695	TYPE d = vd, n = vn, *m = vm; \
696	uint8_t *pg = vg; \
697	for (i = 0; i < opr_sz; i += 2) { \
698	TYPE n0 = n[i], n1 = n[i + 1]; \
699	TYPE m0 = m[i], m1 = m[i + 1]; \
700	if (pg[H1(i)] & 1) { \
701	d[i] = OP(n0, n1); \
702	} \
703	if (pg[H1(i + 1)] & 1) { \
704	d[i + 1] = OP(m0, m1); \
705	} \
706	} \
707	}
708
709	DO_ZPZZ_PAIR(sve2_addp_zpzz_b, uint8_t, H1, DO_ADD)
710	DO_ZPZZ_PAIR(sve2_addp_zpzz_h, uint16_t, H1_2, DO_ADD)
711	DO_ZPZZ_PAIR(sve2_addp_zpzz_s, uint32_t, H1_4, DO_ADD)
712	DO_ZPZZ_PAIR_D(sve2_addp_zpzz_d, uint64_t, DO_ADD)
713
714	DO_ZPZZ_PAIR(sve2_umaxp_zpzz_b, uint8_t, H1, DO_MAX)
715	DO_ZPZZ_PAIR(sve2_umaxp_zpzz_h, uint16_t, H1_2, DO_MAX)
716	DO_ZPZZ_PAIR(sve2_umaxp_zpzz_s, uint32_t, H1_4, DO_MAX)
717	DO_ZPZZ_PAIR_D(sve2_umaxp_zpzz_d, uint64_t, DO_MAX)
718
719	DO_ZPZZ_PAIR(sve2_uminp_zpzz_b, uint8_t, H1, DO_MIN)
720	DO_ZPZZ_PAIR(sve2_uminp_zpzz_h, uint16_t, H1_2, DO_MIN)
721	DO_ZPZZ_PAIR(sve2_uminp_zpzz_s, uint32_t, H1_4, DO_MIN)
722	DO_ZPZZ_PAIR_D(sve2_uminp_zpzz_d, uint64_t, DO_MIN)
723
724	DO_ZPZZ_PAIR(sve2_smaxp_zpzz_b, int8_t, H1, DO_MAX)
725	DO_ZPZZ_PAIR(sve2_smaxp_zpzz_h, int16_t, H1_2, DO_MAX)
726	DO_ZPZZ_PAIR(sve2_smaxp_zpzz_s, int32_t, H1_4, DO_MAX)
727	DO_ZPZZ_PAIR_D(sve2_smaxp_zpzz_d, int64_t, DO_MAX)
728
729	DO_ZPZZ_PAIR(sve2_sminp_zpzz_b, int8_t, H1, DO_MIN)
730	DO_ZPZZ_PAIR(sve2_sminp_zpzz_h, int16_t, H1_2, DO_MIN)
731	DO_ZPZZ_PAIR(sve2_sminp_zpzz_s, int32_t, H1_4, DO_MIN)
732	DO_ZPZZ_PAIR_D(sve2_sminp_zpzz_d, int64_t, DO_MIN)
733
734	#undef DO_ZPZZ_PAIR
735	#undef DO_ZPZZ_PAIR_D
736
737	#define DO_ZPZZ_PAIR_FP(NAME, TYPE, H, OP) \
738	void HELPER(NAME)(void vd, void vn, void vm, void vg, \
739	float_status *status, uint32_t desc) \
740	{ \
741	intptr_t i, opr_sz = simd_oprsz(desc); \
742	for (i = 0; i < opr_sz; ) { \
743	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
744	do { \
745	TYPE n0 = (TYPE )(vn + H(i)); \
746	TYPE m0 = (TYPE )(vm + H(i)); \
747	TYPE n1 = (TYPE )(vn + H(i + sizeof(TYPE))); \
748	TYPE m1 = (TYPE )(vm + H(i + sizeof(TYPE))); \
749	if (pg & 1) { \
750	(TYPE )(vd + H(i)) = OP(n0, n1, status); \
751	} \
752	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
753	if (pg & 1) { \
754	(TYPE )(vd + H(i)) = OP(m0, m1, status); \
755	} \
756	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
757	} while (i & 15); \
758	} \
759	}
760
761	DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_h, float16, H1_2, float16_add)
762	DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_s, float32, H1_4, float32_add)
763	DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_d, float64, H1_8, float64_add)
764
765	DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_h, float16, H1_2, float16_maxnum)
766	DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_s, float32, H1_4, float32_maxnum)
767	DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_d, float64, H1_8, float64_maxnum)
768
769	DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_h, float16, H1_2, float16_minnum)
770	DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_s, float32, H1_4, float32_minnum)
771	DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_d, float64, H1_8, float64_minnum)
772
773	DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_h, float16, H1_2, float16_max)
774	DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_s, float32, H1_4, float32_max)
775	DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_d, float64, H1_8, float64_max)
776
777	DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_h, float16, H1_2, float16_min)
778	DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_s, float32, H1_4, float32_min)
779	DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_d, float64, H1_8, float64_min)
780
781	DO_ZPZZ_PAIR_FP(sve2_ah_fmaxp_zpzz_h, float16, H1_2, helper_vfp_ah_maxh)
782	DO_ZPZZ_PAIR_FP(sve2_ah_fmaxp_zpzz_s, float32, H1_4, helper_vfp_ah_maxs)
783	DO_ZPZZ_PAIR_FP(sve2_ah_fmaxp_zpzz_d, float64, H1_8, helper_vfp_ah_maxd)
784
785	DO_ZPZZ_PAIR_FP(sve2_ah_fminp_zpzz_h, float16, H1_2, helper_vfp_ah_minh)
786	DO_ZPZZ_PAIR_FP(sve2_ah_fminp_zpzz_s, float32, H1_4, helper_vfp_ah_mins)
787	DO_ZPZZ_PAIR_FP(sve2_ah_fminp_zpzz_d, float64, H1_8, helper_vfp_ah_mind)
788
789	#undef DO_ZPZZ_PAIR_FP
790
791	/* Three-operand expander, controlled by a predicate, in which the
792	* third operand is "wide". That is, for D = N op M, the same 64-bit
793	* value of M is used with all of the narrower values of N.
794	*/
795	#define DO_ZPZW(NAME, TYPE, TYPEW, H, OP) \
796	void HELPER(NAME)(void vd, void vn, void vm, void vg, uint32_t desc) \
797	{ \
798	intptr_t i, opr_sz = simd_oprsz(desc); \
799	for (i = 0; i < opr_sz; ) { \
800	uint8_t pg = (uint8_t )(vg + H1(i >> 3)); \
801	TYPEW mm = (TYPEW )(vm + i); \
802	do { \
803	if (pg & 1) { \
804	TYPE nn = (TYPE )(vn + H(i)); \
805	(TYPE )(vd + H(i)) = OP(nn, mm); \
806	} \
807	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
808	} while (i & 7); \
809	} \
810	}
811
812	DO_ZPZW(sve_asr_zpzw_b, int8_t, uint64_t, H1, DO_ASR)
813	DO_ZPZW(sve_lsr_zpzw_b, uint8_t, uint64_t, H1, DO_LSR)
814	DO_ZPZW(sve_lsl_zpzw_b, uint8_t, uint64_t, H1, DO_LSL)
815
816	DO_ZPZW(sve_asr_zpzw_h, int16_t, uint64_t, H1_2, DO_ASR)
817	DO_ZPZW(sve_lsr_zpzw_h, uint16_t, uint64_t, H1_2, DO_LSR)
818	DO_ZPZW(sve_lsl_zpzw_h, uint16_t, uint64_t, H1_2, DO_LSL)
819
820	DO_ZPZW(sve_asr_zpzw_s, int32_t, uint64_t, H1_4, DO_ASR)
821	DO_ZPZW(sve_lsr_zpzw_s, uint32_t, uint64_t, H1_4, DO_LSR)
822	DO_ZPZW(sve_lsl_zpzw_s, uint32_t, uint64_t, H1_4, DO_LSL)
823
824	#undef DO_ZPZW
825
826	/* Fully general two-operand expander, controlled by a predicate. */
827	#define DO_ZPZ(NAME, TYPE, H, OP) \
828	void HELPER(NAME)(void vd, void vn, void *vg, uint32_t desc) \
829	{ \
830	intptr_t i, opr_sz = simd_oprsz(desc); \
831	bool zeroing = simd_data(desc) & 1; \
832	for (i = 0; i < opr_sz; ) { \
833	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
834	do { \
835	if (pg & 1) { \
836	TYPE nn = (TYPE )(vn + H(i)); \
837	(TYPE )(vd + H(i)) = OP(nn); \
838	} else if (zeroing) { \
839	(TYPE )(vd + H(i)) = 0; \
840	} \
841	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
842	} while (i & 15); \
843	} \
844	}
845
846	/* Similarly, specialized for 64-bit operands. */
847	#define DO_ZPZ_D(NAME, TYPE, OP) \
848	void HELPER(NAME)(void vd, void vn, void *vg, uint32_t desc) \
849	{ \
850	intptr_t i, opr_sz = simd_oprsz(desc) / 8; \
851	bool zeroing = simd_data(desc) & 1; \
852	TYPE d = vd, n = vn; \
853	uint8_t *pg = vg; \
854	for (i = 0; i < opr_sz; i += 1) { \
855	if (pg[H1(i)] & 1) { \
856	TYPE nn = n[i]; \
857	d[i] = OP(nn); \
858	} else if (zeroing) { \
859	d[i] = 0; \
860	} \
861	} \
862	}
863
864	#define DO_CLS_B(N) (clrsb32(N) - 24)
865	#define DO_CLS_H(N) (clrsb32(N) - 16)
866
867	DO_ZPZ(sve_cls_b, int8_t, H1, DO_CLS_B)
868	DO_ZPZ(sve_cls_h, int16_t, H1_2, DO_CLS_H)
869	DO_ZPZ(sve_cls_s, int32_t, H1_4, clrsb32)
870	DO_ZPZ_D(sve_cls_d, int64_t, clrsb64)
871
872	#define DO_CLZ_B(N) (clz32(N) - 24)
873	#define DO_CLZ_H(N) (clz32(N) - 16)
874
875	DO_ZPZ(sve_clz_b, uint8_t, H1, DO_CLZ_B)
876	DO_ZPZ(sve_clz_h, uint16_t, H1_2, DO_CLZ_H)
877	DO_ZPZ(sve_clz_s, uint32_t, H1_4, clz32)
878	DO_ZPZ_D(sve_clz_d, uint64_t, clz64)
879
880	DO_ZPZ(sve_cnt_zpz_b, uint8_t, H1, ctpop8)
881	DO_ZPZ(sve_cnt_zpz_h, uint16_t, H1_2, ctpop16)
882	DO_ZPZ(sve_cnt_zpz_s, uint32_t, H1_4, ctpop32)
883	DO_ZPZ_D(sve_cnt_zpz_d, uint64_t, ctpop64)
884
885	#define DO_CNOT(N) (N == 0)
886
887	DO_ZPZ(sve_cnot_b, uint8_t, H1, DO_CNOT)
888	DO_ZPZ(sve_cnot_h, uint16_t, H1_2, DO_CNOT)
889	DO_ZPZ(sve_cnot_s, uint32_t, H1_4, DO_CNOT)
890	DO_ZPZ_D(sve_cnot_d, uint64_t, DO_CNOT)
891
892	#define DO_FABS(N) (N & ((__typeof(N))-1 >> 1))
893
894	DO_ZPZ(sve_fabs_h, uint16_t, H1_2, DO_FABS)
895	DO_ZPZ(sve_fabs_s, uint32_t, H1_4, DO_FABS)
896	DO_ZPZ_D(sve_fabs_d, uint64_t, DO_FABS)
897
898	#define DO_AH_FABS_H(N) (float16_is_any_nan(N) ? (N) : DO_FABS(N))
899	#define DO_AH_FABS_S(N) (float32_is_any_nan(N) ? (N) : DO_FABS(N))
900	#define DO_AH_FABS_D(N) (float64_is_any_nan(N) ? (N) : DO_FABS(N))
901
902	DO_ZPZ(sve_ah_fabs_h, uint16_t, H1_2, DO_AH_FABS_H)
903	DO_ZPZ(sve_ah_fabs_s, uint32_t, H1_4, DO_AH_FABS_S)
904	DO_ZPZ_D(sve_ah_fabs_d, uint64_t, DO_AH_FABS_D)
905
906	#define DO_FNEG(N) (N ^ ~((__typeof(N))-1 >> 1))
907
908	DO_ZPZ(sve_fneg_h, uint16_t, H1_2, DO_FNEG)
909	DO_ZPZ(sve_fneg_s, uint32_t, H1_4, DO_FNEG)
910	DO_ZPZ_D(sve_fneg_d, uint64_t, DO_FNEG)
911
912	#define DO_AH_FNEG_H(N) (float16_is_any_nan(N) ? (N) : DO_FNEG(N))
913	#define DO_AH_FNEG_S(N) (float32_is_any_nan(N) ? (N) : DO_FNEG(N))
914	#define DO_AH_FNEG_D(N) (float64_is_any_nan(N) ? (N) : DO_FNEG(N))
915
916	DO_ZPZ(sve_ah_fneg_h, uint16_t, H1_2, DO_AH_FNEG_H)
917	DO_ZPZ(sve_ah_fneg_s, uint32_t, H1_4, DO_AH_FNEG_S)
918	DO_ZPZ_D(sve_ah_fneg_d, uint64_t, DO_AH_FNEG_D)
919
920	static inline void
921	sve_not_zpz(uint64_t d, uint64_t n, uint8_t *pg, uint32_t desc,
922	uint64_t (*expand)(uint8_t))
923	{
924	intptr_t opr_sz = simd_oprsz(desc) / 8;
925	bool zeroing = simd_data(desc) & 1;
926
927	if (zeroing) {
928	for (intptr_t i = 0; i < opr_sz; ++i) {
929	uint64_t p = expand(pg[H1(i)]);
930	d[i] = ~n[i] & p;
931	}
932	} else {
933	for (intptr_t i = 0; i < opr_sz; ++i) {
934	uint64_t p = expand(pg[H1(i)]);
935	d[i] = (~n[i] & p) \| (d[i] & ~p);
936	}
937	}
938	}
939
940	void HELPER(sve_not_zpz_b)(void vd, void vn, void *pg, uint32_t desc)
941	{
942	sve_not_zpz(vd, vn, pg, desc, expand_pred_b);
943	}
944
945	void HELPER(sve_not_zpz_h)(void vd, void vn, void *pg, uint32_t desc)
946	{
947	sve_not_zpz(vd, vn, pg, desc, expand_pred_h);
948	}
949
950	void HELPER(sve_not_zpz_s)(void vd, void vn, void *pg, uint32_t desc)
951	{
952	sve_not_zpz(vd, vn, pg, desc, expand_pred_s);
953	}
954
955	void HELPER(sve_not_zpz_d)(void vd, void vn, void *pg, uint32_t desc)
956	{
957	sve_not_zpz(vd, vn, pg, desc, expand_pred_d);
958	}
959
960	#define DO_SXTB(N) ((int8_t)N)
961	#define DO_SXTH(N) ((int16_t)N)
962	#define DO_SXTS(N) ((int32_t)N)
963	#define DO_UXTB(N) ((uint8_t)N)
964	#define DO_UXTH(N) ((uint16_t)N)
965	#define DO_UXTS(N) ((uint32_t)N)
966
967	DO_ZPZ(sve_sxtb_h, uint16_t, H1_2, DO_SXTB)
968	DO_ZPZ(sve_sxtb_s, uint32_t, H1_4, DO_SXTB)
969	DO_ZPZ(sve_sxth_s, uint32_t, H1_4, DO_SXTH)
970	DO_ZPZ_D(sve_sxtb_d, uint64_t, DO_SXTB)
971	DO_ZPZ_D(sve_sxth_d, uint64_t, DO_SXTH)
972	DO_ZPZ_D(sve_sxtw_d, uint64_t, DO_SXTS)
973
974	DO_ZPZ(sve_uxtb_h, uint16_t, H1_2, DO_UXTB)
975	DO_ZPZ(sve_uxtb_s, uint32_t, H1_4, DO_UXTB)
976	DO_ZPZ(sve_uxth_s, uint32_t, H1_4, DO_UXTH)
977	DO_ZPZ_D(sve_uxtb_d, uint64_t, DO_UXTB)
978	DO_ZPZ_D(sve_uxth_d, uint64_t, DO_UXTH)
979	DO_ZPZ_D(sve_uxtw_d, uint64_t, DO_UXTS)
980
981	#define DO_ABS(N) (N < 0 ? -N : N)
982
983	DO_ZPZ(sve_abs_b, int8_t, H1, DO_ABS)
984	DO_ZPZ(sve_abs_h, int16_t, H1_2, DO_ABS)
985	DO_ZPZ(sve_abs_s, int32_t, H1_4, DO_ABS)
986	DO_ZPZ_D(sve_abs_d, int64_t, DO_ABS)
987
988	#define DO_NEG(N) (-N)
989
990	DO_ZPZ(sve_neg_b, uint8_t, H1, DO_NEG)
991	DO_ZPZ(sve_neg_h, uint16_t, H1_2, DO_NEG)
992	DO_ZPZ(sve_neg_s, uint32_t, H1_4, DO_NEG)
993	DO_ZPZ_D(sve_neg_d, uint64_t, DO_NEG)
994
995	DO_ZPZ(sve_revb_h, uint16_t, H1_2, bswap16)
996	DO_ZPZ(sve_revb_s, uint32_t, H1_4, bswap32)
997	DO_ZPZ_D(sve_revb_d, uint64_t, bswap64)
998
999	DO_ZPZ(sve_revh_s, uint32_t, H1_4, hswap32)
1000	DO_ZPZ_D(sve_revh_d, uint64_t, hswap64)
1001
1002	DO_ZPZ_D(sve_revw_d, uint64_t, wswap64)
1003
1004	void HELPER(sme_revd_q)(void vd, void vn, void *vg, uint32_t desc)
1005	{
1006	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
1007	bool zeroing = simd_data(desc) & 1;
1008	uint64_t d = vd, n = vn;
1009	uint8_t *pg = vg;
1010
1011	for (i = 0; i < opr_sz; i += 2) {
1012	if (pg[H1(i)] & 1) {
1013	uint64_t n0 = n[i + 0];
1014	uint64_t n1 = n[i + 1];
1015	d[i + 0] = n1;
1016	d[i + 1] = n0;
1017	} else if (zeroing) {
1018	d[i + 0] = 0;
1019	d[i + 1] = 0;
1020	}
1021	}
1022	}
1023
1024	DO_ZPZ(sve_rbit_b, uint8_t, H1, revbit8)
1025	DO_ZPZ(sve_rbit_h, uint16_t, H1_2, revbit16)
1026	DO_ZPZ(sve_rbit_s, uint32_t, H1_4, revbit32)
1027	DO_ZPZ_D(sve_rbit_d, uint64_t, revbit64)
1028
1029	#define DO_SQABS(X) \
1030	({ __typeof(X) x_ = (X), min_ = 1ull << (sizeof(X) * 8 - 1); \
1031	x_ >= 0 ? x_ : x_ == min_ ? -min_ - 1 : -x_; })
1032
1033	DO_ZPZ(sve2_sqabs_b, int8_t, H1, DO_SQABS)
1034	DO_ZPZ(sve2_sqabs_h, int16_t, H1_2, DO_SQABS)
1035	DO_ZPZ(sve2_sqabs_s, int32_t, H1_4, DO_SQABS)
1036	DO_ZPZ_D(sve2_sqabs_d, int64_t, DO_SQABS)
1037
1038	#define DO_SQNEG(X) \
1039	({ __typeof(X) x_ = (X), min_ = 1ull << (sizeof(X) * 8 - 1); \
1040	x_ == min_ ? -min_ - 1 : -x_; })
1041
1042	DO_ZPZ(sve2_sqneg_b, uint8_t, H1, DO_SQNEG)
1043	DO_ZPZ(sve2_sqneg_h, uint16_t, H1_2, DO_SQNEG)
1044	DO_ZPZ(sve2_sqneg_s, uint32_t, H1_4, DO_SQNEG)
1045	DO_ZPZ_D(sve2_sqneg_d, uint64_t, DO_SQNEG)
1046
1047	DO_ZPZ(sve2_urecpe_s, uint32_t, H1_4, helper_recpe_u32)
1048	DO_ZPZ(sve2_ursqrte_s, uint32_t, H1_4, helper_rsqrte_u32)
1049
1050	/* Three-operand expander, unpredicated, in which the third operand is "wide".
1051	*/
1052	#define DO_ZZW(NAME, TYPE, TYPEW, H, OP) \
1053	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
1054	{ \
1055	intptr_t i, opr_sz = simd_oprsz(desc); \
1056	for (i = 0; i < opr_sz; ) { \
1057	TYPEW mm = (TYPEW )(vm + i); \
1058	do { \
1059	TYPE nn = (TYPE )(vn + H(i)); \
1060	(TYPE )(vd + H(i)) = OP(nn, mm); \
1061	i += sizeof(TYPE); \
1062	} while (i & 7); \
1063	} \
1064	}
1065
1066	DO_ZZW(sve_asr_zzw_b, int8_t, uint64_t, H1, DO_ASR)
1067	DO_ZZW(sve_lsr_zzw_b, uint8_t, uint64_t, H1, DO_LSR)
1068	DO_ZZW(sve_lsl_zzw_b, uint8_t, uint64_t, H1, DO_LSL)
1069
1070	DO_ZZW(sve_asr_zzw_h, int16_t, uint64_t, H1_2, DO_ASR)
1071	DO_ZZW(sve_lsr_zzw_h, uint16_t, uint64_t, H1_2, DO_LSR)
1072	DO_ZZW(sve_lsl_zzw_h, uint16_t, uint64_t, H1_2, DO_LSL)
1073
1074	DO_ZZW(sve_asr_zzw_s, int32_t, uint64_t, H1_4, DO_ASR)
1075	DO_ZZW(sve_lsr_zzw_s, uint32_t, uint64_t, H1_4, DO_LSR)
1076	DO_ZZW(sve_lsl_zzw_s, uint32_t, uint64_t, H1_4, DO_LSL)
1077
1078	#undef DO_ZZW
1079
1080	#undef DO_CLS_B
1081	#undef DO_CLS_H
1082	#undef DO_CLZ_B
1083	#undef DO_CLZ_H
1084	#undef DO_CNOT
1085	#undef DO_FABS
1086	#undef DO_FNEG
1087	#undef DO_ABS
1088	#undef DO_NEG
1089	#undef DO_ZPZ
1090	#undef DO_ZPZ_D
1091
1092	/*
1093	* Three-operand expander, unpredicated, in which the two inputs are
1094	* selected from the top or bottom half of the wide column.
1095	*/
1096	#define DO_ZZZ_TB(NAME, TYPEW, TYPEN, HW, HN, OP) \
1097	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
1098	{ \
1099	intptr_t i, opr_sz = simd_oprsz(desc); \
1100	int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
1101	int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \
1102	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
1103	TYPEW nn = (TYPEN )(vn + HN(i + sel1)); \
1104	TYPEW mm = (TYPEN )(vm + HN(i + sel2)); \
1105	(TYPEW )(vd + HW(i)) = OP(nn, mm); \
1106	} \
1107	}
1108
1109	DO_ZZZ_TB(sve2_saddl_h, int16_t, int8_t, H1_2, H1, DO_ADD)
1110	DO_ZZZ_TB(sve2_saddl_s, int32_t, int16_t, H1_4, H1_2, DO_ADD)
1111	DO_ZZZ_TB(sve2_saddl_d, int64_t, int32_t, H1_8, H1_4, DO_ADD)
1112
1113	DO_ZZZ_TB(sve2_ssubl_h, int16_t, int8_t, H1_2, H1, DO_SUB)
1114	DO_ZZZ_TB(sve2_ssubl_s, int32_t, int16_t, H1_4, H1_2, DO_SUB)
1115	DO_ZZZ_TB(sve2_ssubl_d, int64_t, int32_t, H1_8, H1_4, DO_SUB)
1116
1117	DO_ZZZ_TB(sve2_sabdl_h, int16_t, int8_t, H1_2, H1, DO_ABD)
1118	DO_ZZZ_TB(sve2_sabdl_s, int32_t, int16_t, H1_4, H1_2, DO_ABD)
1119	DO_ZZZ_TB(sve2_sabdl_d, int64_t, int32_t, H1_8, H1_4, DO_ABD)
1120
1121	DO_ZZZ_TB(sve2_uaddl_h, uint16_t, uint8_t, H1_2, H1, DO_ADD)
1122	DO_ZZZ_TB(sve2_uaddl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD)
1123	DO_ZZZ_TB(sve2_uaddl_d, uint64_t, uint32_t, H1_8, H1_4, DO_ADD)
1124
1125	DO_ZZZ_TB(sve2_usubl_h, uint16_t, uint8_t, H1_2, H1, DO_SUB)
1126	DO_ZZZ_TB(sve2_usubl_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB)
1127	DO_ZZZ_TB(sve2_usubl_d, uint64_t, uint32_t, H1_8, H1_4, DO_SUB)
1128
1129	DO_ZZZ_TB(sve2_uabdl_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
1130	DO_ZZZ_TB(sve2_uabdl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
1131	DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, H1_8, H1_4, DO_ABD)
1132
1133	DO_ZZZ_TB(sve2_smull_zzz_h, int16_t, int8_t, H1_2, H1, DO_MUL)
1134	DO_ZZZ_TB(sve2_smull_zzz_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
1135	DO_ZZZ_TB(sve2_smull_zzz_d, int64_t, int32_t, H1_8, H1_4, DO_MUL)
1136
1137	DO_ZZZ_TB(sve2_umull_zzz_h, uint16_t, uint8_t, H1_2, H1, DO_MUL)
1138	DO_ZZZ_TB(sve2_umull_zzz_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
1139	DO_ZZZ_TB(sve2_umull_zzz_d, uint64_t, uint32_t, H1_8, H1_4, DO_MUL)
1140
1141	/* Note that the multiply cannot overflow, but the doubling can. */
1142	static inline int16_t do_sqdmull_h(int16_t n, int16_t m)
1143	{
1144	int16_t val = n * m;
1145	return DO_SQADD_H(val, val);
1146	}
1147
1148	static inline int32_t do_sqdmull_s(int32_t n, int32_t m)
1149	{
1150	int32_t val = n * m;
1151	return DO_SQADD_S(val, val);
1152	}
1153
1154	static inline int64_t do_sqdmull_d(int64_t n, int64_t m)
1155	{
1156	int64_t val = n * m;
1157	return do_sqadd_d(val, val);
1158	}
1159
1160	DO_ZZZ_TB(sve2_sqdmull_zzz_h, int16_t, int8_t, H1_2, H1, do_sqdmull_h)
1161	DO_ZZZ_TB(sve2_sqdmull_zzz_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s)
1162	DO_ZZZ_TB(sve2_sqdmull_zzz_d, int64_t, int32_t, H1_8, H1_4, do_sqdmull_d)
1163
1164	#undef DO_ZZZ_TB
1165
1166	#define DO_ZZZ_WTB(NAME, TYPEW, TYPEN, HW, HN, OP) \
1167	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
1168	{ \
1169	intptr_t i, opr_sz = simd_oprsz(desc); \
1170	int sel2 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
1171	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
1172	TYPEW nn = (TYPEW )(vn + HW(i)); \
1173	TYPEW mm = (TYPEN )(vm + HN(i + sel2)); \
1174	(TYPEW )(vd + HW(i)) = OP(nn, mm); \
1175	} \
1176	}
1177
1178	DO_ZZZ_WTB(sve2_saddw_h, int16_t, int8_t, H1_2, H1, DO_ADD)
1179	DO_ZZZ_WTB(sve2_saddw_s, int32_t, int16_t, H1_4, H1_2, DO_ADD)
1180	DO_ZZZ_WTB(sve2_saddw_d, int64_t, int32_t, H1_8, H1_4, DO_ADD)
1181
1182	DO_ZZZ_WTB(sve2_ssubw_h, int16_t, int8_t, H1_2, H1, DO_SUB)
1183	DO_ZZZ_WTB(sve2_ssubw_s, int32_t, int16_t, H1_4, H1_2, DO_SUB)
1184	DO_ZZZ_WTB(sve2_ssubw_d, int64_t, int32_t, H1_8, H1_4, DO_SUB)
1185
1186	DO_ZZZ_WTB(sve2_uaddw_h, uint16_t, uint8_t, H1_2, H1, DO_ADD)
1187	DO_ZZZ_WTB(sve2_uaddw_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD)
1188	DO_ZZZ_WTB(sve2_uaddw_d, uint64_t, uint32_t, H1_8, H1_4, DO_ADD)
1189
1190	DO_ZZZ_WTB(sve2_usubw_h, uint16_t, uint8_t, H1_2, H1, DO_SUB)
1191	DO_ZZZ_WTB(sve2_usubw_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB)
1192	DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, H1_8, H1_4, DO_SUB)
1193
1194	#undef DO_ZZZ_WTB
1195
1196	#define DO_ZZZ_NTB(NAME, TYPE, H, OP) \
1197	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
1198	{ \
1199	intptr_t i, opr_sz = simd_oprsz(desc); \
1200	intptr_t sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPE); \
1201	intptr_t sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPE); \
1202	for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \
1203	TYPE nn = (TYPE )(vn + H(i + sel1)); \
1204	TYPE mm = (TYPE )(vm + H(i + sel2)); \
1205	(TYPE )(vd + H(i + sel1)) = OP(nn, mm); \
1206	} \
1207	}
1208
1209	DO_ZZZ_NTB(sve2_eoril_b, uint8_t, H1, DO_EOR)
1210	DO_ZZZ_NTB(sve2_eoril_h, uint16_t, H1_2, DO_EOR)
1211	DO_ZZZ_NTB(sve2_eoril_s, uint32_t, H1_4, DO_EOR)
1212	DO_ZZZ_NTB(sve2_eoril_d, uint64_t, H1_8, DO_EOR)
1213
1214	#undef DO_ZZZ_NTB
1215
1216	#define DO_ZZZW_ACC(NAME, TYPEW, TYPEN, HW, HN, OP) \
1217	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
1218	{ \
1219	intptr_t i, opr_sz = simd_oprsz(desc); \
1220	intptr_t sel1 = simd_data(desc) * sizeof(TYPEN); \
1221	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
1222	TYPEW nn = (TYPEN )(vn + HN(i + sel1)); \
1223	TYPEW mm = (TYPEN )(vm + HN(i + sel1)); \
1224	TYPEW aa = (TYPEW )(va + HW(i)); \
1225	(TYPEW )(vd + HW(i)) = OP(nn, mm) + aa; \
1226	} \
1227	}
1228
1229	DO_ZZZW_ACC(sve2_sabal_h, int16_t, int8_t, H1_2, H1, DO_ABD)
1230	DO_ZZZW_ACC(sve2_sabal_s, int32_t, int16_t, H1_4, H1_2, DO_ABD)
1231	DO_ZZZW_ACC(sve2_sabal_d, int64_t, int32_t, H1_8, H1_4, DO_ABD)
1232
1233	DO_ZZZW_ACC(sve2_uabal_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
1234	DO_ZZZW_ACC(sve2_uabal_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
1235	DO_ZZZW_ACC(sve2_uabal_d, uint64_t, uint32_t, H1_8, H1_4, DO_ABD)
1236
1237	DO_ZZZW_ACC(sve2_smlal_zzzw_h, int16_t, int8_t, H1_2, H1, DO_MUL)
1238	DO_ZZZW_ACC(sve2_smlal_zzzw_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
1239	DO_ZZZW_ACC(sve2_smlal_zzzw_d, int64_t, int32_t, H1_8, H1_4, DO_MUL)
1240
1241	DO_ZZZW_ACC(sve2_umlal_zzzw_h, uint16_t, uint8_t, H1_2, H1, DO_MUL)
1242	DO_ZZZW_ACC(sve2_umlal_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
1243	DO_ZZZW_ACC(sve2_umlal_zzzw_d, uint64_t, uint32_t, H1_8, H1_4, DO_MUL)
1244
1245	#define DO_NMUL(N, M) -(N * M)
1246
1247	DO_ZZZW_ACC(sve2_smlsl_zzzw_h, int16_t, int8_t, H1_2, H1, DO_NMUL)
1248	DO_ZZZW_ACC(sve2_smlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2, DO_NMUL)
1249	DO_ZZZW_ACC(sve2_smlsl_zzzw_d, int64_t, int32_t, H1_8, H1_4, DO_NMUL)
1250
1251	DO_ZZZW_ACC(sve2_umlsl_zzzw_h, uint16_t, uint8_t, H1_2, H1, DO_NMUL)
1252	DO_ZZZW_ACC(sve2_umlsl_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, DO_NMUL)
1253	DO_ZZZW_ACC(sve2_umlsl_zzzw_d, uint64_t, uint32_t, H1_8, H1_4, DO_NMUL)
1254
1255	#undef DO_ZZZW_ACC
1256
1257	#define DO_XTNB(NAME, TYPE, OP) \
1258	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
1259	{ \
1260	intptr_t i, opr_sz = simd_oprsz(desc); \
1261	for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
1262	TYPE nn = (TYPE )(vn + i); \
1263	nn = OP(nn) & MAKE_64BIT_MASK(0, sizeof(TYPE) * 4); \
1264	(TYPE )(vd + i) = nn; \
1265	} \
1266	}
1267
1268	#define DO_XTNT(NAME, TYPE, TYPEN, H, OP) \
1269	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
1270	{ \
1271	intptr_t i, opr_sz = simd_oprsz(desc), odd = H(sizeof(TYPEN)); \
1272	for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
1273	TYPE nn = (TYPE )(vn + i); \
1274	(TYPEN )(vd + i + odd) = OP(nn); \
1275	} \
1276	}
1277
1278	DO_XTNB(sve2_sqxtnb_h, int16_t, do_ssat_b)
1279	DO_XTNB(sve2_sqxtnb_s, int32_t, do_ssat_h)
1280	DO_XTNB(sve2_sqxtnb_d, int64_t, do_ssat_s)
1281
1282	DO_XTNT(sve2_sqxtnt_h, int16_t, int8_t, H1, do_ssat_b)
1283	DO_XTNT(sve2_sqxtnt_s, int32_t, int16_t, H1_2, do_ssat_h)
1284	DO_XTNT(sve2_sqxtnt_d, int64_t, int32_t, H1_4, do_ssat_s)
1285
1286	DO_XTNB(sve2_uqxtnb_h, uint16_t, do_usat_b)
1287	DO_XTNB(sve2_uqxtnb_s, uint32_t, do_usat_h)
1288	DO_XTNB(sve2_uqxtnb_d, uint64_t, do_usat_s)
1289
1290	DO_XTNT(sve2_uqxtnt_h, uint16_t, uint8_t, H1, do_usat_b)
1291	DO_XTNT(sve2_uqxtnt_s, uint32_t, uint16_t, H1_2, do_usat_h)
1292	DO_XTNT(sve2_uqxtnt_d, uint64_t, uint32_t, H1_4, do_usat_s)
1293
1294	DO_XTNB(sve2_sqxtunb_h, int16_t, do_usat_b)
1295	DO_XTNB(sve2_sqxtunb_s, int32_t, do_usat_h)
1296	DO_XTNB(sve2_sqxtunb_d, int64_t, do_usat_s)
1297
1298	DO_XTNT(sve2_sqxtunt_h, int16_t, int8_t, H1, do_usat_b)
1299	DO_XTNT(sve2_sqxtunt_s, int32_t, int16_t, H1_2, do_usat_h)
1300	DO_XTNT(sve2_sqxtunt_d, int64_t, int32_t, H1_4, do_usat_s)
1301
1302	#undef DO_XTNB
1303	#undef DO_XTNT
1304
1305	void HELPER(sve2_adcl_s)(void vd, void vn, void vm, void va, uint32_t desc)
1306	{
1307	intptr_t i, opr_sz = simd_oprsz(desc);
1308	int sel = H4(extract32(desc, SIMD_DATA_SHIFT, 1));
1309	uint32_t inv = -extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1310	uint32_t a = va, n = vn;
1311	uint64_t d = vd, m = vm;
1312
1313	for (i = 0; i < opr_sz / 8; ++i) {
1314	uint32_t e1 = a[2 * i + H4(0)];
1315	uint32_t e2 = n[2 * i + sel] ^ inv;
1316	uint64_t c = extract64(m[i], 32, 1);
1317	/* Compute and store the entire 33-bit result at once. */
1318	d[i] = c + e1 + e2;
1319	}
1320	}
1321
1322	void HELPER(sve2_adcl_d)(void vd, void vn, void vm, void va, uint32_t desc)
1323	{
1324	intptr_t i, opr_sz = simd_oprsz(desc);
1325	int sel = extract32(desc, SIMD_DATA_SHIFT, 1);
1326	uint64_t inv = -(uint64_t)extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1327	uint64_t d = vd, a = va, n = vn, m = vm;
1328
1329	for (i = 0; i < opr_sz / 8; i += 2) {
1330	Int128 e1 = int128_make64(a[i]);
1331	Int128 e2 = int128_make64(n[i + sel] ^ inv);
1332	Int128 c = int128_make64(m[i + 1] & 1);
1333	Int128 r = int128_add(int128_add(e1, e2), c);
1334	d[i + 0] = int128_getlo(r);
1335	d[i + 1] = int128_gethi(r);
1336	}
1337	}
1338
1339	#define DO_SQDMLAL(NAME, TYPEW, TYPEN, HW, HN, DMUL_OP, SUM_OP) \
1340	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
1341	{ \
1342	intptr_t i, opr_sz = simd_oprsz(desc); \
1343	int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
1344	int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \
1345	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
1346	TYPEW nn = (TYPEN )(vn + HN(i + sel1)); \
1347	TYPEW mm = (TYPEN )(vm + HN(i + sel2)); \
1348	TYPEW aa = (TYPEW )(va + HW(i)); \
1349	(TYPEW )(vd + HW(i)) = SUM_OP(aa, DMUL_OP(nn, mm)); \
1350	} \
1351	}
1352
1353	DO_SQDMLAL(sve2_sqdmlal_zzzw_h, int16_t, int8_t, H1_2, H1,
1354	do_sqdmull_h, DO_SQADD_H)
1355	DO_SQDMLAL(sve2_sqdmlal_zzzw_s, int32_t, int16_t, H1_4, H1_2,
1356	do_sqdmull_s, DO_SQADD_S)
1357	DO_SQDMLAL(sve2_sqdmlal_zzzw_d, int64_t, int32_t, H1_8, H1_4,
1358	do_sqdmull_d, do_sqadd_d)
1359
1360	DO_SQDMLAL(sve2_sqdmlsl_zzzw_h, int16_t, int8_t, H1_2, H1,
1361	do_sqdmull_h, DO_SQSUB_H)
1362	DO_SQDMLAL(sve2_sqdmlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2,
1363	do_sqdmull_s, DO_SQSUB_S)
1364	DO_SQDMLAL(sve2_sqdmlsl_zzzw_d, int64_t, int32_t, H1_8, H1_4,
1365	do_sqdmull_d, do_sqsub_d)
1366
1367	#undef DO_SQDMLAL
1368
1369	#define DO_CMLA_FUNC(NAME, TYPE, H, OP) \
1370	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
1371	{ \
1372	intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(TYPE); \
1373	int rot = simd_data(desc); \
1374	int sel_a = rot & 1, sel_b = sel_a ^ 1; \
1375	bool sub_r = rot == 1 \|\| rot == 2; \
1376	bool sub_i = rot >= 2; \
1377	TYPE d = vd, n = vn, m = vm, a = va; \
1378	for (i = 0; i < opr_sz; i += 2) { \
1379	TYPE elt1_a = n[H(i + sel_a)]; \
1380	TYPE elt2_a = m[H(i + sel_a)]; \
1381	TYPE elt2_b = m[H(i + sel_b)]; \
1382	d[H(i)] = OP(elt1_a, elt2_a, a[H(i)], sub_r); \
1383	d[H(i + 1)] = OP(elt1_a, elt2_b, a[H(i + 1)], sub_i); \
1384	} \
1385	}
1386
1387	#define DO_CMLA(N, M, A, S) (A + (N * M) * (S ? -1 : 1))
1388
1389	DO_CMLA_FUNC(sve2_cmla_zzzz_b, uint8_t, H1, DO_CMLA)
1390	DO_CMLA_FUNC(sve2_cmla_zzzz_h, uint16_t, H2, DO_CMLA)
1391	DO_CMLA_FUNC(sve2_cmla_zzzz_s, uint32_t, H4, DO_CMLA)
1392	DO_CMLA_FUNC(sve2_cmla_zzzz_d, uint64_t, H8, DO_CMLA)
1393
1394	#define DO_SQRDMLAH_B(N, M, A, S) \
1395	do_sqrdmlah_b(N, M, A, S, true)
1396	#define DO_SQRDMLAH_H(N, M, A, S) \
1397	({ uint32_t discard; do_sqrdmlah_h(N, M, A, S, true, &discard); })
1398	#define DO_SQRDMLAH_S(N, M, A, S) \
1399	({ uint32_t discard; do_sqrdmlah_s(N, M, A, S, true, &discard); })
1400	#define DO_SQRDMLAH_D(N, M, A, S) \
1401	do_sqrdmlah_d(N, M, A, S, true)
1402
1403	DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_b, int8_t, H1, DO_SQRDMLAH_B)
1404	DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_h, int16_t, H2, DO_SQRDMLAH_H)
1405	DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_s, int32_t, H4, DO_SQRDMLAH_S)
1406	DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_d, int64_t, H8, DO_SQRDMLAH_D)
1407
1408	#define DO_CMLA_IDX_FUNC(NAME, TYPE, H, OP) \
1409	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
1410	{ \
1411	intptr_t i, j, oprsz = simd_oprsz(desc); \
1412	int rot = extract32(desc, SIMD_DATA_SHIFT, 2); \
1413	int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2) * 2; \
1414	int sel_a = rot & 1, sel_b = sel_a ^ 1; \
1415	bool sub_r = rot == 1 \|\| rot == 2; \
1416	bool sub_i = rot >= 2; \
1417	TYPE d = vd, n = vn, m = vm, a = va; \
1418	for (i = 0; i < oprsz / sizeof(TYPE); i += 16 / sizeof(TYPE)) { \
1419	TYPE elt2_a = m[H(i + idx + sel_a)]; \
1420	TYPE elt2_b = m[H(i + idx + sel_b)]; \
1421	for (j = 0; j < 16 / sizeof(TYPE); j += 2) { \
1422	TYPE elt1_a = n[H(i + j + sel_a)]; \
1423	d[H2(i + j)] = OP(elt1_a, elt2_a, a[H(i + j)], sub_r); \
1424	d[H2(i + j + 1)] = OP(elt1_a, elt2_b, a[H(i + j + 1)], sub_i); \
1425	} \
1426	} \
1427	}
1428
1429	DO_CMLA_IDX_FUNC(sve2_cmla_idx_h, int16_t, H2, DO_CMLA)
1430	DO_CMLA_IDX_FUNC(sve2_cmla_idx_s, int32_t, H4, DO_CMLA)
1431
1432	DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_h, int16_t, H2, DO_SQRDMLAH_H)
1433	DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S)
1434
1435	#undef DO_CMLA
1436	#undef DO_CMLA_FUNC
1437	#undef DO_CMLA_IDX_FUNC
1438	#undef DO_SQRDMLAH_B
1439	#undef DO_SQRDMLAH_H
1440	#undef DO_SQRDMLAH_S
1441	#undef DO_SQRDMLAH_D
1442
1443	/* Note N and M are 4 elements bundled into one unit. */
1444	static int32_t do_cdot_s(uint32_t n, uint32_t m, int32_t a,
1445	int sel_a, int sel_b, int sub_i)
1446	{
1447	for (int i = 0; i <= 1; i++) {
1448	int32_t elt1_r = (int8_t)(n >> (16 * i));
1449	int32_t elt1_i = (int8_t)(n >> (16 * i + 8));
1450	int32_t elt2_a = (int8_t)(m >> (16 * i + 8 * sel_a));
1451	int32_t elt2_b = (int8_t)(m >> (16 * i + 8 * sel_b));
1452
1453	a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
1454	}
1455	return a;
1456	}
1457
1458	static int64_t do_cdot_d(uint64_t n, uint64_t m, int64_t a,
1459	int sel_a, int sel_b, int sub_i)
1460	{
1461	for (int i = 0; i <= 1; i++) {
1462	int64_t elt1_r = (int16_t)(n >> (32 * i + 0));
1463	int64_t elt1_i = (int16_t)(n >> (32 * i + 16));
1464	int64_t elt2_a = (int16_t)(m >> (32 * i + 16 * sel_a));
1465	int64_t elt2_b = (int16_t)(m >> (32 * i + 16 * sel_b));
1466
1467	a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
1468	}
1469	return a;
1470	}
1471
1472	void HELPER(sve2_cdot_zzzz_s)(void vd, void vn, void *vm,
1473	void *va, uint32_t desc)
1474	{
1475	int opr_sz = simd_oprsz(desc);
1476	int rot = simd_data(desc);
1477	int sel_a = rot & 1;
1478	int sel_b = sel_a ^ 1;
1479	int sub_i = (rot == 0 \|\| rot == 3 ? -1 : 1);
1480	uint32_t d = vd, n = vn, m = vm, a = va;
1481
1482	for (int e = 0; e < opr_sz / 4; e++) {
1483	d[e] = do_cdot_s(n[e], m[e], a[e], sel_a, sel_b, sub_i);
1484	}
1485	}
1486
1487	void HELPER(sve2_cdot_zzzz_d)(void vd, void vn, void *vm,
1488	void *va, uint32_t desc)
1489	{
1490	int opr_sz = simd_oprsz(desc);
1491	int rot = simd_data(desc);
1492	int sel_a = rot & 1;
1493	int sel_b = sel_a ^ 1;
1494	int sub_i = (rot == 0 \|\| rot == 3 ? -1 : 1);
1495	uint64_t d = vd, n = vn, m = vm, a = va;
1496
1497	for (int e = 0; e < opr_sz / 8; e++) {
1498	d[e] = do_cdot_d(n[e], m[e], a[e], sel_a, sel_b, sub_i);
1499	}
1500	}
1501
1502	void HELPER(sve2_cdot_idx_s)(void vd, void vn, void *vm,
1503	void *va, uint32_t desc)
1504	{
1505	int opr_sz = simd_oprsz(desc);
1506	int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
1507	int idx = H4(extract32(desc, SIMD_DATA_SHIFT + 2, 2));
1508	int sel_a = rot & 1;
1509	int sel_b = sel_a ^ 1;
1510	int sub_i = (rot == 0 \|\| rot == 3 ? -1 : 1);
1511	uint32_t d = vd, n = vn, m = vm, a = va;
1512
1513	for (int seg = 0; seg < opr_sz / 4; seg += 4) {
1514	uint32_t seg_m = m[seg + idx];
1515	for (int e = 0; e < 4; e++) {
1516	d[seg + e] = do_cdot_s(n[seg + e], seg_m, a[seg + e],
1517	sel_a, sel_b, sub_i);
1518	}
1519	}
1520	}
1521
1522	void HELPER(sve2_cdot_idx_d)(void vd, void vn, void *vm,
1523	void *va, uint32_t desc)
1524	{
1525	int seg, opr_sz = simd_oprsz(desc);
1526	int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
1527	int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
1528	int sel_a = rot & 1;
1529	int sel_b = sel_a ^ 1;
1530	int sub_i = (rot == 0 \|\| rot == 3 ? -1 : 1);
1531	uint64_t d = vd, n = vn, m = vm, a = va;
1532
1533	for (seg = 0; seg < opr_sz / 8; seg += 2) {
1534	uint64_t seg_m = m[seg + idx];
1535	for (int e = 0; e < 2; e++) {
1536	d[seg + e] = do_cdot_d(n[seg + e], seg_m, a[seg + e],
1537	sel_a, sel_b, sub_i);
1538	}
1539	}
1540	}
1541
1542	#define DO_ZZXZ(NAME, TYPE, H, OP) \
1543	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
1544	{ \
1545	intptr_t oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE); \
1546	intptr_t i, j, idx = simd_data(desc); \
1547	TYPE d = vd, a = va, n = vn, m = (TYPE *)vm + H(idx); \
1548	for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1549	TYPE mm = m[i]; \
1550	for (j = 0; j < segment; j++) { \
1551	d[i + j] = OP(n[i + j], mm, a[i + j]); \
1552	} \
1553	} \
1554	}
1555
1556	#define DO_SQRDMLAH_H(N, M, A) \
1557	({ uint32_t discard; do_sqrdmlah_h(N, M, A, false, true, &discard); })
1558	#define DO_SQRDMLAH_S(N, M, A) \
1559	({ uint32_t discard; do_sqrdmlah_s(N, M, A, false, true, &discard); })
1560	#define DO_SQRDMLAH_D(N, M, A) do_sqrdmlah_d(N, M, A, false, true)
1561
1562	DO_ZZXZ(sve2_sqrdmlah_idx_h, int16_t, H2, DO_SQRDMLAH_H)
1563	DO_ZZXZ(sve2_sqrdmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S)
1564	DO_ZZXZ(sve2_sqrdmlah_idx_d, int64_t, H8, DO_SQRDMLAH_D)
1565
1566	#define DO_SQRDMLSH_H(N, M, A) \
1567	({ uint32_t discard; do_sqrdmlah_h(N, M, A, true, true, &discard); })
1568	#define DO_SQRDMLSH_S(N, M, A) \
1569	({ uint32_t discard; do_sqrdmlah_s(N, M, A, true, true, &discard); })
1570	#define DO_SQRDMLSH_D(N, M, A) do_sqrdmlah_d(N, M, A, true, true)
1571
1572	DO_ZZXZ(sve2_sqrdmlsh_idx_h, int16_t, H2, DO_SQRDMLSH_H)
1573	DO_ZZXZ(sve2_sqrdmlsh_idx_s, int32_t, H4, DO_SQRDMLSH_S)
1574	DO_ZZXZ(sve2_sqrdmlsh_idx_d, int64_t, H8, DO_SQRDMLSH_D)
1575
1576	#undef DO_ZZXZ
1577
1578	#define DO_ZZXW(NAME, TYPEW, TYPEN, HW, HN, OP) \
1579	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
1580	{ \
1581	intptr_t i, j, oprsz = simd_oprsz(desc); \
1582	intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
1583	intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 1, 3) * sizeof(TYPEN); \
1584	for (i = 0; i < oprsz; i += 16) { \
1585	TYPEW mm = (TYPEN )(vm + HN(i + idx)); \
1586	for (j = 0; j < 16; j += sizeof(TYPEW)) { \
1587	TYPEW nn = (TYPEN )(vn + HN(i + j + sel)); \
1588	TYPEW aa = (TYPEW )(va + HW(i + j)); \
1589	(TYPEW )(vd + HW(i + j)) = OP(nn, mm, aa); \
1590	} \
1591	} \
1592	}
1593
1594	#define DO_MLA(N, M, A) (A + N * M)
1595
1596	DO_ZZXW(sve2_smlal_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MLA)
1597	DO_ZZXW(sve2_smlal_idx_d, int64_t, int32_t, H1_8, H1_4, DO_MLA)
1598	DO_ZZXW(sve2_umlal_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MLA)
1599	DO_ZZXW(sve2_umlal_idx_d, uint64_t, uint32_t, H1_8, H1_4, DO_MLA)
1600
1601	#define DO_MLS(N, M, A) (A - N * M)
1602
1603	DO_ZZXW(sve2_smlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MLS)
1604	DO_ZZXW(sve2_smlsl_idx_d, int64_t, int32_t, H1_8, H1_4, DO_MLS)
1605	DO_ZZXW(sve2_umlsl_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MLS)
1606	DO_ZZXW(sve2_umlsl_idx_d, uint64_t, uint32_t, H1_8, H1_4, DO_MLS)
1607
1608	#define DO_SQDMLAL_S(N, M, A) DO_SQADD_S(A, do_sqdmull_s(N, M))
1609	#define DO_SQDMLAL_D(N, M, A) do_sqadd_d(A, do_sqdmull_d(N, M))
1610
1611	DO_ZZXW(sve2_sqdmlal_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLAL_S)
1612	DO_ZZXW(sve2_sqdmlal_idx_d, int64_t, int32_t, H1_8, H1_4, DO_SQDMLAL_D)
1613
1614	#define DO_SQDMLSL_S(N, M, A) DO_SQSUB_S(A, do_sqdmull_s(N, M))
1615	#define DO_SQDMLSL_D(N, M, A) do_sqsub_d(A, do_sqdmull_d(N, M))
1616
1617	DO_ZZXW(sve2_sqdmlsl_idx_s, int32_t, int16_t, H1_4, H1_2, DO_SQDMLSL_S)
1618	DO_ZZXW(sve2_sqdmlsl_idx_d, int64_t, int32_t, H1_8, H1_4, DO_SQDMLSL_D)
1619
1620	#undef DO_MLA
1621	#undef DO_MLS
1622	#undef DO_ZZXW
1623
1624	#define DO_ZZX(NAME, TYPEW, TYPEN, HW, HN, OP) \
1625	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
1626	{ \
1627	intptr_t i, j, oprsz = simd_oprsz(desc); \
1628	intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN); \
1629	intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 1, 3) * sizeof(TYPEN); \
1630	for (i = 0; i < oprsz; i += 16) { \
1631	TYPEW mm = (TYPEN )(vm + HN(i + idx)); \
1632	for (j = 0; j < 16; j += sizeof(TYPEW)) { \
1633	TYPEW nn = (TYPEN )(vn + HN(i + j + sel)); \
1634	(TYPEW )(vd + HW(i + j)) = OP(nn, mm); \
1635	} \
1636	} \
1637	}
1638
1639	DO_ZZX(sve2_sqdmull_idx_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s)
1640	DO_ZZX(sve2_sqdmull_idx_d, int64_t, int32_t, H1_8, H1_4, do_sqdmull_d)
1641
1642	DO_ZZX(sve2_smull_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
1643	DO_ZZX(sve2_smull_idx_d, int64_t, int32_t, H1_8, H1_4, DO_MUL)
1644
1645	DO_ZZX(sve2_umull_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
1646	DO_ZZX(sve2_umull_idx_d, uint64_t, uint32_t, H1_8, H1_4, DO_MUL)
1647
1648	#undef DO_ZZX
1649
1650	#define DO_BITPERM(NAME, TYPE, OP) \
1651	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
1652	{ \
1653	intptr_t i, opr_sz = simd_oprsz(desc); \
1654	for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
1655	TYPE nn = (TYPE )(vn + i); \
1656	TYPE mm = (TYPE )(vm + i); \
1657	(TYPE )(vd + i) = OP(nn, mm, sizeof(TYPE) * 8); \
1658	} \
1659	}
1660
1661	static uint64_t bitextract(uint64_t data, uint64_t mask, int n)
1662	{
1663	uint64_t res = 0;
1664	int db, rb = 0;
1665
1666	for (db = 0; db < n; ++db) {
1667	if ((mask >> db) & 1) {
1668	res \|= ((data >> db) & 1) << rb;
1669	++rb;
1670	}
1671	}
1672	return res;
1673	}
1674
1675	DO_BITPERM(sve2_bext_b, uint8_t, bitextract)
1676	DO_BITPERM(sve2_bext_h, uint16_t, bitextract)
1677	DO_BITPERM(sve2_bext_s, uint32_t, bitextract)
1678	DO_BITPERM(sve2_bext_d, uint64_t, bitextract)
1679
1680	static uint64_t bitdeposit(uint64_t data, uint64_t mask, int n)
1681	{
1682	uint64_t res = 0;
1683	int rb, db = 0;
1684
1685	for (rb = 0; rb < n; ++rb) {
1686	if ((mask >> rb) & 1) {
1687	res \|= ((data >> db) & 1) << rb;
1688	++db;
1689	}
1690	}
1691	return res;
1692	}
1693
1694	DO_BITPERM(sve2_bdep_b, uint8_t, bitdeposit)
1695	DO_BITPERM(sve2_bdep_h, uint16_t, bitdeposit)
1696	DO_BITPERM(sve2_bdep_s, uint32_t, bitdeposit)
1697	DO_BITPERM(sve2_bdep_d, uint64_t, bitdeposit)
1698
1699	static uint64_t bitgroup(uint64_t data, uint64_t mask, int n)
1700	{
1701	uint64_t resm = 0, resu = 0;
1702	int db, rbm = 0, rbu = 0;
1703
1704	for (db = 0; db < n; ++db) {
1705	uint64_t val = (data >> db) & 1;
1706	if ((mask >> db) & 1) {
1707	resm \|= val << rbm++;
1708	} else {
1709	resu \|= val << rbu++;
1710	}
1711	}
1712
1713	return resm \| (resu << rbm);
1714	}
1715
1716	DO_BITPERM(sve2_bgrp_b, uint8_t, bitgroup)
1717	DO_BITPERM(sve2_bgrp_h, uint16_t, bitgroup)
1718	DO_BITPERM(sve2_bgrp_s, uint32_t, bitgroup)
1719	DO_BITPERM(sve2_bgrp_d, uint64_t, bitgroup)
1720
1721	#undef DO_BITPERM
1722
1723	#define DO_CADD(NAME, TYPE, H, ADD_OP, SUB_OP) \
1724	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
1725	{ \
1726	intptr_t i, opr_sz = simd_oprsz(desc); \
1727	int sub_r = simd_data(desc); \
1728	if (sub_r) { \
1729	for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \
1730	TYPE acc_r = (TYPE )(vn + H(i)); \
1731	TYPE acc_i = (TYPE )(vn + H(i + sizeof(TYPE))); \
1732	TYPE el2_r = (TYPE )(vm + H(i)); \
1733	TYPE el2_i = (TYPE )(vm + H(i + sizeof(TYPE))); \
1734	acc_r = ADD_OP(acc_r, el2_i); \
1735	acc_i = SUB_OP(acc_i, el2_r); \
1736	(TYPE )(vd + H(i)) = acc_r; \
1737	(TYPE )(vd + H(i + sizeof(TYPE))) = acc_i; \
1738	} \
1739	} else { \
1740	for (i = 0; i < opr_sz; i += 2 * sizeof(TYPE)) { \
1741	TYPE acc_r = (TYPE )(vn + H(i)); \
1742	TYPE acc_i = (TYPE )(vn + H(i + sizeof(TYPE))); \
1743	TYPE el2_r = (TYPE )(vm + H(i)); \
1744	TYPE el2_i = (TYPE )(vm + H(i + sizeof(TYPE))); \
1745	acc_r = SUB_OP(acc_r, el2_i); \
1746	acc_i = ADD_OP(acc_i, el2_r); \
1747	(TYPE )(vd + H(i)) = acc_r; \
1748	(TYPE )(vd + H(i + sizeof(TYPE))) = acc_i; \
1749	} \
1750	} \
1751	}
1752
1753	DO_CADD(sve2_cadd_b, int8_t, H1, DO_ADD, DO_SUB)
1754	DO_CADD(sve2_cadd_h, int16_t, H1_2, DO_ADD, DO_SUB)
1755	DO_CADD(sve2_cadd_s, int32_t, H1_4, DO_ADD, DO_SUB)
1756	DO_CADD(sve2_cadd_d, int64_t, H1_8, DO_ADD, DO_SUB)
1757
1758	DO_CADD(sve2_sqcadd_b, int8_t, H1, DO_SQADD_B, DO_SQSUB_B)
1759	DO_CADD(sve2_sqcadd_h, int16_t, H1_2, DO_SQADD_H, DO_SQSUB_H)
1760	DO_CADD(sve2_sqcadd_s, int32_t, H1_4, DO_SQADD_S, DO_SQSUB_S)
1761	DO_CADD(sve2_sqcadd_d, int64_t, H1_8, do_sqadd_d, do_sqsub_d)
1762
1763	#undef DO_CADD
1764
1765	#define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN) \
1766	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
1767	{ \
1768	intptr_t i, opr_sz = simd_oprsz(desc); \
1769	intptr_t sel = (simd_data(desc) & 1) * sizeof(TYPEN); \
1770	int shift = simd_data(desc) >> 1; \
1771	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
1772	TYPEW nn = (TYPEN )(vn + HN(i + sel)); \
1773	(TYPEW )(vd + HW(i)) = nn << shift; \
1774	} \
1775	}
1776
1777	DO_ZZI_SHLL(sve2_sshll_h, int16_t, int8_t, H1_2, H1)
1778	DO_ZZI_SHLL(sve2_sshll_s, int32_t, int16_t, H1_4, H1_2)
1779	DO_ZZI_SHLL(sve2_sshll_d, int64_t, int32_t, H1_8, H1_4)
1780
1781	DO_ZZI_SHLL(sve2_ushll_h, uint16_t, uint8_t, H1_2, H1)
1782	DO_ZZI_SHLL(sve2_ushll_s, uint32_t, uint16_t, H1_4, H1_2)
1783	DO_ZZI_SHLL(sve2_ushll_d, uint64_t, uint32_t, H1_8, H1_4)
1784
1785	#undef DO_ZZI_SHLL
1786
1787	/* Two-operand reduction expander, controlled by a predicate.
1788	* The difference between TYPERED and TYPERET has to do with
1789	* sign-extension. E.g. for SMAX, TYPERED must be signed,
1790	* but TYPERET must be unsigned so that e.g. a 32-bit value
1791	* is not sign-extended to the ABI uint64_t return type.
1792	*/
1793	/* ??? If we were to vectorize this by hand the reduction ordering
1794	* would change. For integer operands, this is perfectly fine.
1795	*/
1796	#define DO_VPZ(NAME, TYPEELT, TYPERED, TYPERET, H, INIT, OP) \
1797	uint64_t HELPER(NAME)(void vn, void vg, uint32_t desc) \
1798	{ \
1799	intptr_t i, opr_sz = simd_oprsz(desc); \
1800	TYPERED ret = INIT; \
1801	for (i = 0; i < opr_sz; ) { \
1802	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
1803	do { \
1804	if (pg & 1) { \
1805	TYPEELT nn = (TYPEELT )(vn + H(i)); \
1806	ret = OP(ret, nn); \
1807	} \
1808	i += sizeof(TYPEELT), pg >>= sizeof(TYPEELT); \
1809	} while (i & 15); \
1810	} \
1811	return (TYPERET)ret; \
1812	}
1813
1814	#define DO_VPZ_D(NAME, TYPEE, TYPER, INIT, OP) \
1815	uint64_t HELPER(NAME)(void vn, void vg, uint32_t desc) \
1816	{ \
1817	intptr_t i, opr_sz = simd_oprsz(desc) / 8; \
1818	TYPEE *n = vn; \
1819	uint8_t *pg = vg; \
1820	TYPER ret = INIT; \
1821	for (i = 0; i < opr_sz; i += 1) { \
1822	if (pg[H1(i)] & 1) { \
1823	TYPEE nn = n[i]; \
1824	ret = OP(ret, nn); \
1825	} \
1826	} \
1827	return ret; \
1828	}
1829
1830	DO_VPZ(sve_orv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_ORR)
1831	DO_VPZ(sve_orv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_ORR)
1832	DO_VPZ(sve_orv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_ORR)
1833	DO_VPZ_D(sve_orv_d, uint64_t, uint64_t, 0, DO_ORR)
1834
1835	DO_VPZ(sve_eorv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_EOR)
1836	DO_VPZ(sve_eorv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_EOR)
1837	DO_VPZ(sve_eorv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_EOR)
1838	DO_VPZ_D(sve_eorv_d, uint64_t, uint64_t, 0, DO_EOR)
1839
1840	DO_VPZ(sve_andv_b, uint8_t, uint8_t, uint8_t, H1, -1, DO_AND)
1841	DO_VPZ(sve_andv_h, uint16_t, uint16_t, uint16_t, H1_2, -1, DO_AND)
1842	DO_VPZ(sve_andv_s, uint32_t, uint32_t, uint32_t, H1_4, -1, DO_AND)
1843	DO_VPZ_D(sve_andv_d, uint64_t, uint64_t, -1, DO_AND)
1844
1845	DO_VPZ(sve_saddv_b, int8_t, uint64_t, uint64_t, H1, 0, DO_ADD)
1846	DO_VPZ(sve_saddv_h, int16_t, uint64_t, uint64_t, H1_2, 0, DO_ADD)
1847	DO_VPZ(sve_saddv_s, int32_t, uint64_t, uint64_t, H1_4, 0, DO_ADD)
1848
1849	DO_VPZ(sve_uaddv_b, uint8_t, uint64_t, uint64_t, H1, 0, DO_ADD)
1850	DO_VPZ(sve_uaddv_h, uint16_t, uint64_t, uint64_t, H1_2, 0, DO_ADD)
1851	DO_VPZ(sve_uaddv_s, uint32_t, uint64_t, uint64_t, H1_4, 0, DO_ADD)
1852	DO_VPZ_D(sve_uaddv_d, uint64_t, uint64_t, 0, DO_ADD)
1853
1854	DO_VPZ(sve_smaxv_b, int8_t, int8_t, uint8_t, H1, INT8_MIN, DO_MAX)
1855	DO_VPZ(sve_smaxv_h, int16_t, int16_t, uint16_t, H1_2, INT16_MIN, DO_MAX)
1856	DO_VPZ(sve_smaxv_s, int32_t, int32_t, uint32_t, H1_4, INT32_MIN, DO_MAX)
1857	DO_VPZ_D(sve_smaxv_d, int64_t, int64_t, INT64_MIN, DO_MAX)
1858
1859	DO_VPZ(sve_umaxv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_MAX)
1860	DO_VPZ(sve_umaxv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_MAX)
1861	DO_VPZ(sve_umaxv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_MAX)
1862	DO_VPZ_D(sve_umaxv_d, uint64_t, uint64_t, 0, DO_MAX)
1863
1864	DO_VPZ(sve_sminv_b, int8_t, int8_t, uint8_t, H1, INT8_MAX, DO_MIN)
1865	DO_VPZ(sve_sminv_h, int16_t, int16_t, uint16_t, H1_2, INT16_MAX, DO_MIN)
1866	DO_VPZ(sve_sminv_s, int32_t, int32_t, uint32_t, H1_4, INT32_MAX, DO_MIN)
1867	DO_VPZ_D(sve_sminv_d, int64_t, int64_t, INT64_MAX, DO_MIN)
1868
1869	DO_VPZ(sve_uminv_b, uint8_t, uint8_t, uint8_t, H1, -1, DO_MIN)
1870	DO_VPZ(sve_uminv_h, uint16_t, uint16_t, uint16_t, H1_2, -1, DO_MIN)
1871	DO_VPZ(sve_uminv_s, uint32_t, uint32_t, uint32_t, H1_4, -1, DO_MIN)
1872	DO_VPZ_D(sve_uminv_d, uint64_t, uint64_t, -1, DO_MIN)
1873
1874	#undef DO_VPZ
1875	#undef DO_VPZ_D
1876
1877	#define DO_VPQ(NAME, TYPE, H, INIT, OP) \
1878	void HELPER(NAME)(void vd, void vn, void *vg, uint32_t desc) \
1879	{ \
1880	TYPE tmp[16 / sizeof(TYPE)] = { [0 ... 16 / sizeof(TYPE) - 1] = INIT }; \
1881	TYPE n = vn; uint16_t g = vg; \
1882	uintptr_t oprsz = simd_oprsz(desc); \
1883	uintptr_t nseg = oprsz / 16, nsegelt = 16 / sizeof(TYPE); \
1884	for (uintptr_t s = 0; s < nseg; s++) { \
1885	uint16_t pg = g[H2(s)]; \
1886	for (uintptr_t e = 0; e < nsegelt; e++, pg >>= sizeof(TYPE)) { \
1887	if (pg & 1) { \
1888	tmp[e] = OP(tmp[H(e)], n[s * nsegelt + H(e)]); \
1889	} \
1890	} \
1891	} \
1892	memcpy(vd, tmp, 16); \
1893	clear_tail(vd, 16, simd_maxsz(desc)); \
1894	}
1895
1896	DO_VPQ(sve2p1_addqv_b, uint8_t, H1, 0, DO_ADD)
1897	DO_VPQ(sve2p1_addqv_h, uint16_t, H2, 0, DO_ADD)
1898	DO_VPQ(sve2p1_addqv_s, uint32_t, H4, 0, DO_ADD)
1899	DO_VPQ(sve2p1_addqv_d, uint64_t, H8, 0, DO_ADD)
1900
1901	DO_VPQ(sve2p1_smaxqv_b, int8_t, H1, INT8_MIN, DO_MAX)
1902	DO_VPQ(sve2p1_smaxqv_h, int16_t, H2, INT16_MIN, DO_MAX)
1903	DO_VPQ(sve2p1_smaxqv_s, int32_t, H4, INT32_MIN, DO_MAX)
1904	DO_VPQ(sve2p1_smaxqv_d, int64_t, H8, INT64_MIN, DO_MAX)
1905
1906	DO_VPQ(sve2p1_sminqv_b, int8_t, H1, INT8_MAX, DO_MIN)
1907	DO_VPQ(sve2p1_sminqv_h, int16_t, H2, INT16_MAX, DO_MIN)
1908	DO_VPQ(sve2p1_sminqv_s, int32_t, H4, INT32_MAX, DO_MIN)
1909	DO_VPQ(sve2p1_sminqv_d, int64_t, H8, INT64_MAX, DO_MIN)
1910
1911	DO_VPQ(sve2p1_umaxqv_b, uint8_t, H1, 0, DO_MAX)
1912	DO_VPQ(sve2p1_umaxqv_h, uint16_t, H2, 0, DO_MAX)
1913	DO_VPQ(sve2p1_umaxqv_s, uint32_t, H4, 0, DO_MAX)
1914	DO_VPQ(sve2p1_umaxqv_d, uint64_t, H8, 0, DO_MAX)
1915
1916	DO_VPQ(sve2p1_uminqv_b, uint8_t, H1, -1, DO_MIN)
1917	DO_VPQ(sve2p1_uminqv_h, uint16_t, H2, -1, DO_MIN)
1918	DO_VPQ(sve2p1_uminqv_s, uint32_t, H4, -1, DO_MIN)
1919	DO_VPQ(sve2p1_uminqv_d, uint64_t, H8, -1, DO_MIN)
1920
1921	#undef DO_VPQ
1922
1923	/* Two vector operand, one scalar operand, unpredicated. */
1924	#define DO_ZZI(NAME, TYPE, OP) \
1925	void HELPER(NAME)(void vd, void vn, uint64_t s64, uint32_t desc) \
1926	{ \
1927	intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(TYPE); \
1928	TYPE s = s64, d = vd, n = vn; \
1929	for (i = 0; i < opr_sz; ++i) { \
1930	d[i] = OP(n[i], s); \
1931	} \
1932	}
1933
1934	#define DO_SUBR(X, Y) (Y - X)
1935
1936	DO_ZZI(sve_subri_b, uint8_t, DO_SUBR)
1937	DO_ZZI(sve_subri_h, uint16_t, DO_SUBR)
1938	DO_ZZI(sve_subri_s, uint32_t, DO_SUBR)
1939	DO_ZZI(sve_subri_d, uint64_t, DO_SUBR)
1940
1941	DO_ZZI(sve_smaxi_b, int8_t, DO_MAX)
1942	DO_ZZI(sve_smaxi_h, int16_t, DO_MAX)
1943	DO_ZZI(sve_smaxi_s, int32_t, DO_MAX)
1944	DO_ZZI(sve_smaxi_d, int64_t, DO_MAX)
1945
1946	DO_ZZI(sve_smini_b, int8_t, DO_MIN)
1947	DO_ZZI(sve_smini_h, int16_t, DO_MIN)
1948	DO_ZZI(sve_smini_s, int32_t, DO_MIN)
1949	DO_ZZI(sve_smini_d, int64_t, DO_MIN)
1950
1951	DO_ZZI(sve_umaxi_b, uint8_t, DO_MAX)
1952	DO_ZZI(sve_umaxi_h, uint16_t, DO_MAX)
1953	DO_ZZI(sve_umaxi_s, uint32_t, DO_MAX)
1954	DO_ZZI(sve_umaxi_d, uint64_t, DO_MAX)
1955
1956	DO_ZZI(sve_umini_b, uint8_t, DO_MIN)
1957	DO_ZZI(sve_umini_h, uint16_t, DO_MIN)
1958	DO_ZZI(sve_umini_s, uint32_t, DO_MIN)
1959	DO_ZZI(sve_umini_d, uint64_t, DO_MIN)
1960
1961	#undef DO_ZZI
1962
1963	#define DO_LOGIC_QV(NAME, SUFF, INIT, VOP, POP) \
1964	void HELPER(NAME ## _ ## SUFF)(void vd, void vn, void *vg, uint32_t desc) \
1965	{ \
1966	unsigned seg = simd_oprsz(desc) / 16; \
1967	uint64_t r0 = INIT, r1 = INIT; \
1968	for (unsigned s = 0; s < seg; s++) { \
1969	uint64_t p0 = expand_pred_##SUFF((uint8_t )(vg + H1(s * 2))); \
1970	uint64_t p1 = expand_pred_##SUFF((uint8_t )(vg + H1(s * 2 + 1))); \
1971	uint64_t v0 = (uint64_t )(vn + s * 16); \
1972	uint64_t v1 = (uint64_t )(vn + s * 16 + 8); \
1973	v0 = POP(v0, p0), v1 = POP(v1, p1); \
1974	r0 = VOP(r0, v0), r1 = VOP(r1, v1); \
1975	} \
1976	(uint64_t )(vd + 0) = r0; \
1977	(uint64_t )(vd + 8) = r1; \
1978	clear_tail(vd, 16, simd_maxsz(desc)); \
1979	}
1980
1981	DO_LOGIC_QV(sve2p1_orqv, b, 0, DO_ORR, DO_AND)
1982	DO_LOGIC_QV(sve2p1_orqv, h, 0, DO_ORR, DO_AND)
1983	DO_LOGIC_QV(sve2p1_orqv, s, 0, DO_ORR, DO_AND)
1984	DO_LOGIC_QV(sve2p1_orqv, d, 0, DO_ORR, DO_AND)
1985
1986	DO_LOGIC_QV(sve2p1_eorqv, b, 0, DO_EOR, DO_AND)
1987	DO_LOGIC_QV(sve2p1_eorqv, h, 0, DO_EOR, DO_AND)
1988	DO_LOGIC_QV(sve2p1_eorqv, s, 0, DO_EOR, DO_AND)
1989	DO_LOGIC_QV(sve2p1_eorqv, d, 0, DO_EOR, DO_AND)
1990
1991	DO_LOGIC_QV(sve2p1_andqv, b, -1, DO_AND, DO_ORC)
1992	DO_LOGIC_QV(sve2p1_andqv, h, -1, DO_AND, DO_ORC)
1993	DO_LOGIC_QV(sve2p1_andqv, s, -1, DO_AND, DO_ORC)
1994	DO_LOGIC_QV(sve2p1_andqv, d, -1, DO_AND, DO_ORC)
1995
1996	#undef DO_LOGIC_QV
1997
1998	#undef DO_AND
1999	#undef DO_ORR
2000	#undef DO_EOR
2001	#undef DO_BIC
2002	#undef DO_ORC
2003	#undef DO_ADD
2004	#undef DO_SUB
2005	#undef DO_MAX
2006	#undef DO_MIN
2007	#undef DO_ABD
2008	#undef DO_MUL
2009	#undef DO_DIV
2010	#undef DO_ASR
2011	#undef DO_LSR
2012	#undef DO_LSL
2013	#undef DO_SUBR
2014
2015	/* Similar to the ARM LastActiveElement pseudocode function, except the
2016	result is multiplied by the element size. This includes the not found
2017	indication; e.g. not found for esz=3 is -8. */
2018	static intptr_t last_active_element(uint64_t *g, intptr_t words, intptr_t esz)
2019	{
2020	uint64_t mask = pred_esz_masks[esz];
2021	intptr_t i = words;
2022
2023	do {
2024	uint64_t this_g = g[--i] & mask;
2025	if (this_g) {
2026	return i * 64 + (63 - clz64(this_g));
2027	}
2028	} while (i > 0);
2029	return (intptr_t)-1 << esz;
2030	}
2031
2032	uint32_t HELPER(sve_pfirst)(void vd, void vg, uint32_t pred_desc)
2033	{
2034	intptr_t words = DIV_ROUND_UP(FIELD_EX32(pred_desc, PREDDESC, OPRSZ), 8);
2035	uint32_t flags = PREDTEST_INIT;
2036	uint64_t d = vd, g = vg;
2037	intptr_t i = 0;
2038
2039	do {
2040	uint64_t this_d = d[i];
2041	uint64_t this_g = g[i];
2042
2043	if (this_g) {
2044	if (!(flags & 4)) {
2045	/* Set in D the first bit of G. */
2046	this_d \|= this_g & -this_g;
2047	d[i] = this_d;
2048	}
2049	flags = iter_predtest_fwd(this_d, this_g, flags);
2050	}
2051	} while (++i < words);
2052
2053	return flags;
2054	}
2055
2056	uint32_t HELPER(sve_pnext)(void vd, void vg, uint32_t pred_desc)
2057	{
2058	intptr_t words = DIV_ROUND_UP(FIELD_EX32(pred_desc, PREDDESC, OPRSZ), 8);
2059	intptr_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
2060	uint32_t flags = PREDTEST_INIT;
2061	uint64_t d = vd, g = vg, esz_mask;
2062	intptr_t i, next;
2063
2064	next = last_active_element(vd, words, esz) + (1 << esz);
2065	esz_mask = pred_esz_masks[esz];
2066
2067	/* Similar to the pseudocode for pnext, but scaled by ESZ
2068	so that we find the correct bit. */
2069	if (next < words * 64) {
2070	uint64_t mask = -1;
2071
2072	if (next & 63) {
2073	mask = ~((1ull << (next & 63)) - 1);
2074	next &= -64;
2075	}
2076	do {
2077	uint64_t this_g = g[next / 64] & esz_mask & mask;
2078	if (this_g != 0) {
2079	next = (next & -64) + ctz64(this_g);
2080	break;
2081	}
2082	next += 64;
2083	mask = -1;
2084	} while (next < words * 64);
2085	}
2086
2087	i = 0;
2088	do {
2089	uint64_t this_d = 0;
2090	if (i == next / 64) {
2091	this_d = 1ull << (next & 63);
2092	}
2093	d[i] = this_d;
2094	flags = iter_predtest_fwd(this_d, g[i] & esz_mask, flags);
2095	} while (++i < words);
2096
2097	return flags;
2098	}
2099
2100	/*
2101	* Copy Zn into Zd, and store zero into inactive elements.
2102	* If inv, store zeros into the active elements.
2103	*/
2104	void HELPER(sve_movz_b)(void vd, void vn, void *vg, uint32_t desc)
2105	{
2106	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2107	uint64_t inv = -(uint64_t)(simd_data(desc) & 1);
2108	uint64_t d = vd, n = vn;
2109	uint8_t *pg = vg;
2110
2111	for (i = 0; i < opr_sz; i += 1) {
2112	d[i] = n[i] & (expand_pred_b(pg[H1(i)]) ^ inv);
2113	}
2114	}
2115
2116	void HELPER(sve_movz_h)(void vd, void vn, void *vg, uint32_t desc)
2117	{
2118	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2119	uint64_t inv = -(uint64_t)(simd_data(desc) & 1);
2120	uint64_t d = vd, n = vn;
2121	uint8_t *pg = vg;
2122
2123	for (i = 0; i < opr_sz; i += 1) {
2124	d[i] = n[i] & (expand_pred_h(pg[H1(i)]) ^ inv);
2125	}
2126	}
2127
2128	void HELPER(sve_movz_s)(void vd, void vn, void *vg, uint32_t desc)
2129	{
2130	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2131	uint64_t inv = -(uint64_t)(simd_data(desc) & 1);
2132	uint64_t d = vd, n = vn;
2133	uint8_t *pg = vg;
2134
2135	for (i = 0; i < opr_sz; i += 1) {
2136	d[i] = n[i] & (expand_pred_s(pg[H1(i)]) ^ inv);
2137	}
2138	}
2139
2140	void HELPER(sve_movz_d)(void vd, void vn, void *vg, uint32_t desc)
2141	{
2142	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2143	uint64_t d = vd, n = vn;
2144	uint8_t *pg = vg;
2145	uint8_t inv = simd_data(desc);
2146
2147	for (i = 0; i < opr_sz; i += 1) {
2148	d[i] = n[i] & -(uint64_t)((pg[H1(i)] ^ inv) & 1);
2149	}
2150	}
2151
2152	/* Three-operand expander, immediate operand, controlled by a predicate.
2153	*/
2154	#define DO_ZPZI(NAME, TYPE, H, OP) \
2155	void HELPER(NAME)(void vd, void vn, void *vg, uint32_t desc) \
2156	{ \
2157	intptr_t i, opr_sz = simd_oprsz(desc); \
2158	TYPE imm = simd_data(desc); \
2159	for (i = 0; i < opr_sz; ) { \
2160	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
2161	do { \
2162	if (pg & 1) { \
2163	TYPE nn = (TYPE )(vn + H(i)); \
2164	(TYPE )(vd + H(i)) = OP(nn, imm); \
2165	} \
2166	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
2167	} while (i & 15); \
2168	} \
2169	}
2170
2171	/* Similarly, specialized for 64-bit operands. */
2172	#define DO_ZPZI_D(NAME, TYPE, OP) \
2173	void HELPER(NAME)(void vd, void vn, void *vg, uint32_t desc) \
2174	{ \
2175	intptr_t i, opr_sz = simd_oprsz(desc) / 8; \
2176	TYPE d = vd, n = vn; \
2177	TYPE imm = simd_data(desc); \
2178	uint8_t *pg = vg; \
2179	for (i = 0; i < opr_sz; i += 1) { \
2180	if (pg[H1(i)] & 1) { \
2181	TYPE nn = n[i]; \
2182	d[i] = OP(nn, imm); \
2183	} \
2184	} \
2185	}
2186
2187	#define DO_SHR(N, M) (N >> M)
2188	#define DO_SHL(N, M) (N << M)
2189
2190	/* Arithmetic shift right for division. This rounds negative numbers
2191	toward zero as per signed division. Therefore before shifting,
2192	when N is negative, add 2*M-1. /
2193	#define DO_ASRD(N, M) ((N + (N < 0 ? ((__typeof(N))1 << M) - 1 : 0)) >> M)
2194
2195	DO_ZPZI(sve_asr_zpzi_b, int8_t, H1, DO_SHR)
2196	DO_ZPZI(sve_asr_zpzi_h, int16_t, H1_2, DO_SHR)
2197	DO_ZPZI(sve_asr_zpzi_s, int32_t, H1_4, DO_SHR)
2198	DO_ZPZI_D(sve_asr_zpzi_d, int64_t, DO_SHR)
2199
2200	DO_ZPZI(sve_lsr_zpzi_b, uint8_t, H1, DO_SHR)
2201	DO_ZPZI(sve_lsr_zpzi_h, uint16_t, H1_2, DO_SHR)
2202	DO_ZPZI(sve_lsr_zpzi_s, uint32_t, H1_4, DO_SHR)
2203	DO_ZPZI_D(sve_lsr_zpzi_d, uint64_t, DO_SHR)
2204
2205	DO_ZPZI(sve_lsl_zpzi_b, uint8_t, H1, DO_SHL)
2206	DO_ZPZI(sve_lsl_zpzi_h, uint16_t, H1_2, DO_SHL)
2207	DO_ZPZI(sve_lsl_zpzi_s, uint32_t, H1_4, DO_SHL)
2208	DO_ZPZI_D(sve_lsl_zpzi_d, uint64_t, DO_SHL)
2209
2210	DO_ZPZI(sve_asrd_b, int8_t, H1, DO_ASRD)
2211	DO_ZPZI(sve_asrd_h, int16_t, H1_2, DO_ASRD)
2212	DO_ZPZI(sve_asrd_s, int32_t, H1_4, DO_ASRD)
2213	DO_ZPZI_D(sve_asrd_d, int64_t, DO_ASRD)
2214
2215	/* SVE2 bitwise shift by immediate */
2216	DO_ZPZI(sve2_sqshl_zpzi_b, int8_t, H1, do_sqshl_b)
2217	DO_ZPZI(sve2_sqshl_zpzi_h, int16_t, H1_2, do_sqshl_h)
2218	DO_ZPZI(sve2_sqshl_zpzi_s, int32_t, H1_4, do_sqshl_s)
2219	DO_ZPZI_D(sve2_sqshl_zpzi_d, int64_t, do_sqshl_d)
2220
2221	DO_ZPZI(sve2_uqshl_zpzi_b, uint8_t, H1, do_uqshl_b)
2222	DO_ZPZI(sve2_uqshl_zpzi_h, uint16_t, H1_2, do_uqshl_h)
2223	DO_ZPZI(sve2_uqshl_zpzi_s, uint32_t, H1_4, do_uqshl_s)
2224	DO_ZPZI_D(sve2_uqshl_zpzi_d, uint64_t, do_uqshl_d)
2225
2226	DO_ZPZI(sve2_srshr_b, int8_t, H1, do_srshr)
2227	DO_ZPZI(sve2_srshr_h, int16_t, H1_2, do_srshr)
2228	DO_ZPZI(sve2_srshr_s, int32_t, H1_4, do_srshr)
2229	DO_ZPZI_D(sve2_srshr_d, int64_t, do_srshr)
2230
2231	DO_ZPZI(sve2_urshr_b, uint8_t, H1, do_urshr)
2232	DO_ZPZI(sve2_urshr_h, uint16_t, H1_2, do_urshr)
2233	DO_ZPZI(sve2_urshr_s, uint32_t, H1_4, do_urshr)
2234	DO_ZPZI_D(sve2_urshr_d, uint64_t, do_urshr)
2235
2236	#define do_suqrshl_b(n, m) \
2237	({ uint32_t discard; do_suqrshl_bhs(n, (int8_t)m, 8, false, &discard); })
2238	#define do_suqrshl_h(n, m) \
2239	({ uint32_t discard; do_suqrshl_bhs(n, (int16_t)m, 16, false, &discard); })
2240	#define do_suqrshl_s(n, m) \
2241	({ uint32_t discard; do_suqrshl_bhs(n, m, 32, false, &discard); })
2242	#define do_suqrshl_d(n, m) \
2243	({ uint32_t discard; do_suqrshl_d(n, m, false, &discard); })
2244
2245	DO_ZPZI(sve2_sqshlu_b, int8_t, H1, do_suqrshl_b)
2246	DO_ZPZI(sve2_sqshlu_h, int16_t, H1_2, do_suqrshl_h)
2247	DO_ZPZI(sve2_sqshlu_s, int32_t, H1_4, do_suqrshl_s)
2248	DO_ZPZI_D(sve2_sqshlu_d, int64_t, do_suqrshl_d)
2249
2250	#undef DO_ASRD
2251	#undef DO_ZPZI
2252	#undef DO_ZPZI_D
2253
2254	#define DO_SHRNB(NAME, TYPEW, TYPEN, OP) \
2255	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
2256	{ \
2257	intptr_t i, opr_sz = simd_oprsz(desc); \
2258	int shift = simd_data(desc); \
2259	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
2260	TYPEW nn = (TYPEW )(vn + i); \
2261	(TYPEW )(vd + i) = (TYPEN)OP(nn, shift); \
2262	} \
2263	}
2264
2265	#define DO_SHRNT(NAME, TYPEW, TYPEN, HW, HN, OP) \
2266	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
2267	{ \
2268	intptr_t i, opr_sz = simd_oprsz(desc); \
2269	int shift = simd_data(desc); \
2270	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
2271	TYPEW nn = (TYPEW )(vn + HW(i)); \
2272	(TYPEN )(vd + HN(i + sizeof(TYPEN))) = OP(nn, shift); \
2273	} \
2274	}
2275
2276	DO_SHRNB(sve2_shrnb_h, uint16_t, uint8_t, DO_SHR)
2277	DO_SHRNB(sve2_shrnb_s, uint32_t, uint16_t, DO_SHR)
2278	DO_SHRNB(sve2_shrnb_d, uint64_t, uint32_t, DO_SHR)
2279
2280	DO_SHRNT(sve2_shrnt_h, uint16_t, uint8_t, H1_2, H1, DO_SHR)
2281	DO_SHRNT(sve2_shrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_SHR)
2282	DO_SHRNT(sve2_shrnt_d, uint64_t, uint32_t, H1_8, H1_4, DO_SHR)
2283
2284	DO_SHRNB(sve2_rshrnb_h, uint16_t, uint8_t, do_urshr)
2285	DO_SHRNB(sve2_rshrnb_s, uint32_t, uint16_t, do_urshr)
2286	DO_SHRNB(sve2_rshrnb_d, uint64_t, uint32_t, do_urshr)
2287
2288	DO_SHRNT(sve2_rshrnt_h, uint16_t, uint8_t, H1_2, H1, do_urshr)
2289	DO_SHRNT(sve2_rshrnt_s, uint32_t, uint16_t, H1_4, H1_2, do_urshr)
2290	DO_SHRNT(sve2_rshrnt_d, uint64_t, uint32_t, H1_8, H1_4, do_urshr)
2291
2292	#define DO_SQSHRUN_H(x, sh) do_usat_b((int64_t)(x) >> sh)
2293	#define DO_SQSHRUN_S(x, sh) do_usat_h((int64_t)(x) >> sh)
2294	#define DO_SQSHRUN_D(x, sh) do_usat_s((int64_t)(x) >> (sh < 64 ? sh : 63))
2295
2296	DO_SHRNB(sve2_sqshrunb_h, int16_t, uint8_t, DO_SQSHRUN_H)
2297	DO_SHRNB(sve2_sqshrunb_s, int32_t, uint16_t, DO_SQSHRUN_S)
2298	DO_SHRNB(sve2_sqshrunb_d, int64_t, uint32_t, DO_SQSHRUN_D)
2299
2300	DO_SHRNT(sve2_sqshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQSHRUN_H)
2301	DO_SHRNT(sve2_sqshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQSHRUN_S)
2302	DO_SHRNT(sve2_sqshrunt_d, int64_t, uint32_t, H1_8, H1_4, DO_SQSHRUN_D)
2303
2304	#define DO_SQRSHRUN_H(x, sh) do_usat_b(do_srshr(x, sh))
2305	#define DO_SQRSHRUN_S(x, sh) do_usat_h(do_srshr(x, sh))
2306	#define DO_SQRSHRUN_D(x, sh) do_usat_s(do_srshr(x, sh))
2307
2308	DO_SHRNB(sve2_sqrshrunb_h, int16_t, uint8_t, DO_SQRSHRUN_H)
2309	DO_SHRNB(sve2_sqrshrunb_s, int32_t, uint16_t, DO_SQRSHRUN_S)
2310	DO_SHRNB(sve2_sqrshrunb_d, int64_t, uint32_t, DO_SQRSHRUN_D)
2311
2312	DO_SHRNT(sve2_sqrshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRUN_H)
2313	DO_SHRNT(sve2_sqrshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRUN_S)
2314	DO_SHRNT(sve2_sqrshrunt_d, int64_t, uint32_t, H1_8, H1_4, DO_SQRSHRUN_D)
2315
2316	#define DO_SQSHRN_H(x, sh) do_ssat_b(x >> sh)
2317	#define DO_SQSHRN_S(x, sh) do_ssat_h(x >> sh)
2318	#define DO_SQSHRN_D(x, sh) do_ssat_s(x >> sh)
2319
2320	DO_SHRNB(sve2_sqshrnb_h, int16_t, uint8_t, DO_SQSHRN_H)
2321	DO_SHRNB(sve2_sqshrnb_s, int32_t, uint16_t, DO_SQSHRN_S)
2322	DO_SHRNB(sve2_sqshrnb_d, int64_t, uint32_t, DO_SQSHRN_D)
2323
2324	DO_SHRNT(sve2_sqshrnt_h, int16_t, uint8_t, H1_2, H1, DO_SQSHRN_H)
2325	DO_SHRNT(sve2_sqshrnt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQSHRN_S)
2326	DO_SHRNT(sve2_sqshrnt_d, int64_t, uint32_t, H1_8, H1_4, DO_SQSHRN_D)
2327
2328	#define DO_SQRSHRN_H(x, sh) do_ssat_b(do_srshr(x, sh))
2329	#define DO_SQRSHRN_S(x, sh) do_ssat_h(do_srshr(x, sh))
2330	#define DO_SQRSHRN_D(x, sh) do_ssat_s(do_srshr(x, sh))
2331
2332	DO_SHRNB(sve2_sqrshrnb_h, int16_t, uint8_t, DO_SQRSHRN_H)
2333	DO_SHRNB(sve2_sqrshrnb_s, int32_t, uint16_t, DO_SQRSHRN_S)
2334	DO_SHRNB(sve2_sqrshrnb_d, int64_t, uint32_t, DO_SQRSHRN_D)
2335
2336	DO_SHRNT(sve2_sqrshrnt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRN_H)
2337	DO_SHRNT(sve2_sqrshrnt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRN_S)
2338	DO_SHRNT(sve2_sqrshrnt_d, int64_t, uint32_t, H1_8, H1_4, DO_SQRSHRN_D)
2339
2340	#define DO_UQSHRN_H(x, sh) MIN(x >> sh, UINT8_MAX)
2341	#define DO_UQSHRN_S(x, sh) MIN(x >> sh, UINT16_MAX)
2342	#define DO_UQSHRN_D(x, sh) MIN(x >> sh, UINT32_MAX)
2343
2344	DO_SHRNB(sve2_uqshrnb_h, uint16_t, uint8_t, DO_UQSHRN_H)
2345	DO_SHRNB(sve2_uqshrnb_s, uint32_t, uint16_t, DO_UQSHRN_S)
2346	DO_SHRNB(sve2_uqshrnb_d, uint64_t, uint32_t, DO_UQSHRN_D)
2347
2348	DO_SHRNT(sve2_uqshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_UQSHRN_H)
2349	DO_SHRNT(sve2_uqshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_UQSHRN_S)
2350	DO_SHRNT(sve2_uqshrnt_d, uint64_t, uint32_t, H1_8, H1_4, DO_UQSHRN_D)
2351
2352	#define DO_UQRSHRN_H(x, sh) MIN(do_urshr(x, sh), UINT8_MAX)
2353	#define DO_UQRSHRN_S(x, sh) MIN(do_urshr(x, sh), UINT16_MAX)
2354	#define DO_UQRSHRN_D(x, sh) MIN(do_urshr(x, sh), UINT32_MAX)
2355
2356	DO_SHRNB(sve2_uqrshrnb_h, uint16_t, uint8_t, DO_UQRSHRN_H)
2357	DO_SHRNB(sve2_uqrshrnb_s, uint32_t, uint16_t, DO_UQRSHRN_S)
2358	DO_SHRNB(sve2_uqrshrnb_d, uint64_t, uint32_t, DO_UQRSHRN_D)
2359
2360	DO_SHRNT(sve2_uqrshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_UQRSHRN_H)
2361	DO_SHRNT(sve2_uqrshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_UQRSHRN_S)
2362	DO_SHRNT(sve2_uqrshrnt_d, uint64_t, uint32_t, H1_8, H1_4, DO_UQRSHRN_D)
2363
2364	#undef DO_SHRNB
2365	#undef DO_SHRNT
2366
2367	#define DO_BINOPNB(NAME, TYPEW, TYPEN, SHIFT, OP) \
2368	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
2369	{ \
2370	intptr_t i, opr_sz = simd_oprsz(desc); \
2371	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
2372	TYPEW nn = (TYPEW )(vn + i); \
2373	TYPEW mm = (TYPEW )(vm + i); \
2374	(TYPEW )(vd + i) = (TYPEN)OP(nn, mm, SHIFT); \
2375	} \
2376	}
2377
2378	#define DO_BINOPNT(NAME, TYPEW, TYPEN, SHIFT, HW, HN, OP) \
2379	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
2380	{ \
2381	intptr_t i, opr_sz = simd_oprsz(desc); \
2382	for (i = 0; i < opr_sz; i += sizeof(TYPEW)) { \
2383	TYPEW nn = (TYPEW )(vn + HW(i)); \
2384	TYPEW mm = (TYPEW )(vm + HW(i)); \
2385	(TYPEN )(vd + HN(i + sizeof(TYPEN))) = OP(nn, mm, SHIFT); \
2386	} \
2387	}
2388
2389	#define DO_ADDHN(N, M, SH) ((N + M) >> SH)
2390	#define DO_RADDHN(N, M, SH) ((N + M + ((__typeof(N))1 << (SH - 1))) >> SH)
2391	#define DO_SUBHN(N, M, SH) ((N - M) >> SH)
2392	#define DO_RSUBHN(N, M, SH) ((N - M + ((__typeof(N))1 << (SH - 1))) >> SH)
2393
2394	DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, DO_ADDHN)
2395	DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, DO_ADDHN)
2396	DO_BINOPNB(sve2_addhnb_d, uint64_t, uint32_t, 32, DO_ADDHN)
2397
2398	DO_BINOPNT(sve2_addhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_ADDHN)
2399	DO_BINOPNT(sve2_addhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_ADDHN)
2400	DO_BINOPNT(sve2_addhnt_d, uint64_t, uint32_t, 32, H1_8, H1_4, DO_ADDHN)
2401
2402	DO_BINOPNB(sve2_raddhnb_h, uint16_t, uint8_t, 8, DO_RADDHN)
2403	DO_BINOPNB(sve2_raddhnb_s, uint32_t, uint16_t, 16, DO_RADDHN)
2404	DO_BINOPNB(sve2_raddhnb_d, uint64_t, uint32_t, 32, DO_RADDHN)
2405
2406	DO_BINOPNT(sve2_raddhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_RADDHN)
2407	DO_BINOPNT(sve2_raddhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_RADDHN)
2408	DO_BINOPNT(sve2_raddhnt_d, uint64_t, uint32_t, 32, H1_8, H1_4, DO_RADDHN)
2409
2410	DO_BINOPNB(sve2_subhnb_h, uint16_t, uint8_t, 8, DO_SUBHN)
2411	DO_BINOPNB(sve2_subhnb_s, uint32_t, uint16_t, 16, DO_SUBHN)
2412	DO_BINOPNB(sve2_subhnb_d, uint64_t, uint32_t, 32, DO_SUBHN)
2413
2414	DO_BINOPNT(sve2_subhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_SUBHN)
2415	DO_BINOPNT(sve2_subhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_SUBHN)
2416	DO_BINOPNT(sve2_subhnt_d, uint64_t, uint32_t, 32, H1_8, H1_4, DO_SUBHN)
2417
2418	DO_BINOPNB(sve2_rsubhnb_h, uint16_t, uint8_t, 8, DO_RSUBHN)
2419	DO_BINOPNB(sve2_rsubhnb_s, uint32_t, uint16_t, 16, DO_RSUBHN)
2420	DO_BINOPNB(sve2_rsubhnb_d, uint64_t, uint32_t, 32, DO_RSUBHN)
2421
2422	DO_BINOPNT(sve2_rsubhnt_h, uint16_t, uint8_t, 8, H1_2, H1, DO_RSUBHN)
2423	DO_BINOPNT(sve2_rsubhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, DO_RSUBHN)
2424	DO_BINOPNT(sve2_rsubhnt_d, uint64_t, uint32_t, 32, H1_8, H1_4, DO_RSUBHN)
2425
2426	#undef DO_RSUBHN
2427	#undef DO_SUBHN
2428	#undef DO_RADDHN
2429	#undef DO_ADDHN
2430
2431	#undef DO_BINOPNB
2432
2433	/* Fully general four-operand expander, controlled by a predicate.
2434	*/
2435	#define DO_ZPZZZ(NAME, TYPE, H, OP) \
2436	void HELPER(NAME)(void vd, void va, void vn, void vm, \
2437	void *vg, uint32_t desc) \
2438	{ \
2439	intptr_t i, opr_sz = simd_oprsz(desc); \
2440	for (i = 0; i < opr_sz; ) { \
2441	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
2442	do { \
2443	if (pg & 1) { \
2444	TYPE nn = (TYPE )(vn + H(i)); \
2445	TYPE mm = (TYPE )(vm + H(i)); \
2446	TYPE aa = (TYPE )(va + H(i)); \
2447	(TYPE )(vd + H(i)) = OP(aa, nn, mm); \
2448	} \
2449	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
2450	} while (i & 15); \
2451	} \
2452	}
2453
2454	/* Similarly, specialized for 64-bit operands. */
2455	#define DO_ZPZZZ_D(NAME, TYPE, OP) \
2456	void HELPER(NAME)(void vd, void va, void vn, void vm, \
2457	void *vg, uint32_t desc) \
2458	{ \
2459	intptr_t i, opr_sz = simd_oprsz(desc) / 8; \
2460	TYPE d = vd, a = va, n = vn, m = vm; \
2461	uint8_t *pg = vg; \
2462	for (i = 0; i < opr_sz; i += 1) { \
2463	if (pg[H1(i)] & 1) { \
2464	TYPE aa = a[i], nn = n[i], mm = m[i]; \
2465	d[i] = OP(aa, nn, mm); \
2466	} \
2467	} \
2468	}
2469
2470	#define DO_MLA(A, N, M) (A + N * M)
2471	#define DO_MLS(A, N, M) (A - N * M)
2472
2473	DO_ZPZZZ(sve_mla_b, uint8_t, H1, DO_MLA)
2474	DO_ZPZZZ(sve_mls_b, uint8_t, H1, DO_MLS)
2475
2476	DO_ZPZZZ(sve_mla_h, uint16_t, H1_2, DO_MLA)
2477	DO_ZPZZZ(sve_mls_h, uint16_t, H1_2, DO_MLS)
2478
2479	DO_ZPZZZ(sve_mla_s, uint32_t, H1_4, DO_MLA)
2480	DO_ZPZZZ(sve_mls_s, uint32_t, H1_4, DO_MLS)
2481
2482	DO_ZPZZZ_D(sve_mla_d, uint64_t, DO_MLA)
2483	DO_ZPZZZ_D(sve_mls_d, uint64_t, DO_MLS)
2484
2485	#undef DO_MLA
2486	#undef DO_MLS
2487	#undef DO_ZPZZZ
2488	#undef DO_ZPZZZ_D
2489
2490	void HELPER(sve_index_b)(void *vd, uint32_t start,
2491	uint32_t incr, uint32_t desc)
2492	{
2493	intptr_t i, opr_sz = simd_oprsz(desc);
2494	uint8_t *d = vd;
2495	for (i = 0; i < opr_sz; i += 1) {
2496	d[H1(i)] = start + i * incr;
2497	}
2498	}
2499
2500	void HELPER(sve_index_h)(void *vd, uint32_t start,
2501	uint32_t incr, uint32_t desc)
2502	{
2503	intptr_t i, opr_sz = simd_oprsz(desc) / 2;
2504	uint16_t *d = vd;
2505	for (i = 0; i < opr_sz; i += 1) {
2506	d[H2(i)] = start + i * incr;
2507	}
2508	}
2509
2510	void HELPER(sve_index_s)(void *vd, uint32_t start,
2511	uint32_t incr, uint32_t desc)
2512	{
2513	intptr_t i, opr_sz = simd_oprsz(desc) / 4;
2514	uint32_t *d = vd;
2515	for (i = 0; i < opr_sz; i += 1) {
2516	d[H4(i)] = start + i * incr;
2517	}
2518	}
2519
2520	void HELPER(sve_index_d)(void *vd, uint64_t start,
2521	uint64_t incr, uint32_t desc)
2522	{
2523	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2524	uint64_t *d = vd;
2525	for (i = 0; i < opr_sz; i += 1) {
2526	d[i] = start + i * incr;
2527	}
2528	}
2529
2530	void HELPER(sve_adr_p32)(void vd, void vn, void *vm, uint32_t desc)
2531	{
2532	intptr_t i, opr_sz = simd_oprsz(desc) / 4;
2533	uint32_t sh = simd_data(desc);
2534	uint32_t d = vd, n = vn, *m = vm;
2535	for (i = 0; i < opr_sz; i += 1) {
2536	d[i] = n[i] + (m[i] << sh);
2537	}
2538	}
2539
2540	void HELPER(sve_adr_p64)(void vd, void vn, void *vm, uint32_t desc)
2541	{
2542	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2543	uint64_t sh = simd_data(desc);
2544	uint64_t d = vd, n = vn, *m = vm;
2545	for (i = 0; i < opr_sz; i += 1) {
2546	d[i] = n[i] + (m[i] << sh);
2547	}
2548	}
2549
2550	void HELPER(sve_adr_s32)(void vd, void vn, void *vm, uint32_t desc)
2551	{
2552	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2553	uint64_t sh = simd_data(desc);
2554	uint64_t d = vd, n = vn, *m = vm;
2555	for (i = 0; i < opr_sz; i += 1) {
2556	d[i] = n[i] + ((uint64_t)(int32_t)m[i] << sh);
2557	}
2558	}
2559
2560	void HELPER(sve_adr_u32)(void vd, void vn, void *vm, uint32_t desc)
2561	{
2562	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2563	uint64_t sh = simd_data(desc);
2564	uint64_t d = vd, n = vn, *m = vm;
2565	for (i = 0; i < opr_sz; i += 1) {
2566	d[i] = n[i] + ((uint64_t)(uint32_t)m[i] << sh);
2567	}
2568	}
2569
2570	void HELPER(sve_fexpa_h)(void vd, void vn, uint32_t desc)
2571	{
2572	/* These constants are cut-and-paste directly from the ARM pseudocode. */
2573	static const uint16_t coeff[] = {
2574	0x0000, 0x0016, 0x002d, 0x0045, 0x005d, 0x0075, 0x008e, 0x00a8,
2575	0x00c2, 0x00dc, 0x00f8, 0x0114, 0x0130, 0x014d, 0x016b, 0x0189,
2576	0x01a8, 0x01c8, 0x01e8, 0x0209, 0x022b, 0x024e, 0x0271, 0x0295,
2577	0x02ba, 0x02e0, 0x0306, 0x032e, 0x0356, 0x037f, 0x03a9, 0x03d4,
2578	};
2579	intptr_t i, opr_sz = simd_oprsz(desc) / 2;
2580	uint16_t d = vd, n = vn;
2581
2582	for (i = 0; i < opr_sz; i++) {
2583	uint16_t nn = n[i];
2584	intptr_t idx = extract32(nn, 0, 5);
2585	uint16_t exp = extract32(nn, 5, 5);
2586	d[i] = coeff[idx] \| (exp << 10);
2587	}
2588	}
2589
2590	void HELPER(sve_fexpa_s)(void vd, void vn, uint32_t desc)
2591	{
2592	/* These constants are cut-and-paste directly from the ARM pseudocode. */
2593	static const uint32_t coeff[] = {
2594	0x000000, 0x0164d2, 0x02cd87, 0x043a29,
2595	0x05aac3, 0x071f62, 0x08980f, 0x0a14d5,
2596	0x0b95c2, 0x0d1adf, 0x0ea43a, 0x1031dc,
2597	0x11c3d3, 0x135a2b, 0x14f4f0, 0x16942d,
2598	0x1837f0, 0x19e046, 0x1b8d3a, 0x1d3eda,
2599	0x1ef532, 0x20b051, 0x227043, 0x243516,
2600	0x25fed7, 0x27cd94, 0x29a15b, 0x2b7a3a,
2601	0x2d583f, 0x2f3b79, 0x3123f6, 0x3311c4,
2602	0x3504f3, 0x36fd92, 0x38fbaf, 0x3aff5b,
2603	0x3d08a4, 0x3f179a, 0x412c4d, 0x4346cd,
2604	0x45672a, 0x478d75, 0x49b9be, 0x4bec15,
2605	0x4e248c, 0x506334, 0x52a81e, 0x54f35b,
2606	0x5744fd, 0x599d16, 0x5bfbb8, 0x5e60f5,
2607	0x60ccdf, 0x633f89, 0x65b907, 0x68396a,
2608	0x6ac0c7, 0x6d4f30, 0x6fe4ba, 0x728177,
2609	0x75257d, 0x77d0df, 0x7a83b3, 0x7d3e0c,
2610	};
2611	intptr_t i, opr_sz = simd_oprsz(desc) / 4;
2612	uint32_t d = vd, n = vn;
2613
2614	for (i = 0; i < opr_sz; i++) {
2615	uint32_t nn = n[i];
2616	intptr_t idx = extract32(nn, 0, 6);
2617	uint32_t exp = extract32(nn, 6, 8);
2618	d[i] = coeff[idx] \| (exp << 23);
2619	}
2620	}
2621
2622	void HELPER(sve_fexpa_d)(void vd, void vn, uint32_t desc)
2623	{
2624	/* These constants are cut-and-paste directly from the ARM pseudocode. */
2625	static const uint64_t coeff[] = {
2626	0x0000000000000ull, 0x02C9A3E778061ull, 0x059B0D3158574ull,
2627	0x0874518759BC8ull, 0x0B5586CF9890Full, 0x0E3EC32D3D1A2ull,
2628	0x11301D0125B51ull, 0x1429AAEA92DE0ull, 0x172B83C7D517Bull,
2629	0x1A35BEB6FCB75ull, 0x1D4873168B9AAull, 0x2063B88628CD6ull,
2630	0x2387A6E756238ull, 0x26B4565E27CDDull, 0x29E9DF51FDEE1ull,
2631	0x2D285A6E4030Bull, 0x306FE0A31B715ull, 0x33C08B26416FFull,
2632	0x371A7373AA9CBull, 0x3A7DB34E59FF7ull, 0x3DEA64C123422ull,
2633	0x4160A21F72E2Aull, 0x44E086061892Dull, 0x486A2B5C13CD0ull,
2634	0x4BFDAD5362A27ull, 0x4F9B2769D2CA7ull, 0x5342B569D4F82ull,
2635	0x56F4736B527DAull, 0x5AB07DD485429ull, 0x5E76F15AD2148ull,
2636	0x6247EB03A5585ull, 0x6623882552225ull, 0x6A09E667F3BCDull,
2637	0x6DFB23C651A2Full, 0x71F75E8EC5F74ull, 0x75FEB564267C9ull,
2638	0x7A11473EB0187ull, 0x7E2F336CF4E62ull, 0x82589994CCE13ull,
2639	0x868D99B4492EDull, 0x8ACE5422AA0DBull, 0x8F1AE99157736ull,
2640	0x93737B0CDC5E5ull, 0x97D829FDE4E50ull, 0x9C49182A3F090ull,
2641	0xA0C667B5DE565ull, 0xA5503B23E255Dull, 0xA9E6B5579FDBFull,
2642	0xAE89F995AD3ADull, 0xB33A2B84F15FBull, 0xB7F76F2FB5E47ull,
2643	0xBCC1E904BC1D2ull, 0xC199BDD85529Cull, 0xC67F12E57D14Bull,
2644	0xCB720DCEF9069ull, 0xD072D4A07897Cull, 0xD5818DCFBA487ull,
2645	0xDA9E603DB3285ull, 0xDFC97337B9B5Full, 0xE502EE78B3FF6ull,
2646	0xEA4AFA2A490DAull, 0xEFA1BEE615A27ull, 0xF50765B6E4540ull,
2647	0xFA7C1819E90D8ull,
2648	};
2649	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2650	uint64_t d = vd, n = vn;
2651
2652	for (i = 0; i < opr_sz; i++) {
2653	uint64_t nn = n[i];
2654	intptr_t idx = extract32(nn, 0, 6);
2655	uint64_t exp = extract32(nn, 6, 11);
2656	d[i] = coeff[idx] \| (exp << 52);
2657	}
2658	}
2659
2660	void HELPER(sve_ftssel_h)(void vd, void vn, void *vm, uint32_t desc)
2661	{
2662	intptr_t i, opr_sz = simd_oprsz(desc) / 2;
2663	bool fpcr_ah = extract32(desc, SIMD_DATA_SHIFT, 1);
2664	uint16_t d = vd, n = vn, *m = vm;
2665	for (i = 0; i < opr_sz; i += 1) {
2666	uint16_t nn = n[i];
2667	uint16_t mm = m[i];
2668	if (mm & 1) {
2669	nn = float16_one;
2670	}
2671	if (mm & 2) {
2672	nn = float16_maybe_ah_chs(nn, fpcr_ah);
2673	}
2674	d[i] = nn;
2675	}
2676	}
2677
2678	void HELPER(sve_ftssel_s)(void vd, void vn, void *vm, uint32_t desc)
2679	{
2680	intptr_t i, opr_sz = simd_oprsz(desc) / 4;
2681	bool fpcr_ah = extract32(desc, SIMD_DATA_SHIFT, 1);
2682	uint32_t d = vd, n = vn, *m = vm;
2683	for (i = 0; i < opr_sz; i += 1) {
2684	uint32_t nn = n[i];
2685	uint32_t mm = m[i];
2686	if (mm & 1) {
2687	nn = float32_one;
2688	}
2689	if (mm & 2) {
2690	nn = float32_maybe_ah_chs(nn, fpcr_ah);
2691	}
2692	d[i] = nn;
2693	}
2694	}
2695
2696	void HELPER(sve_ftssel_d)(void vd, void vn, void *vm, uint32_t desc)
2697	{
2698	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2699	bool fpcr_ah = extract32(desc, SIMD_DATA_SHIFT, 1);
2700	uint64_t d = vd, n = vn, *m = vm;
2701	for (i = 0; i < opr_sz; i += 1) {
2702	uint64_t nn = n[i];
2703	uint64_t mm = m[i];
2704	if (mm & 1) {
2705	nn = float64_one;
2706	}
2707	if (mm & 2) {
2708	nn = float64_maybe_ah_chs(nn, fpcr_ah);
2709	}
2710	d[i] = nn;
2711	}
2712	}
2713
2714	/*
2715	* Signed saturating addition with scalar operand.
2716	*/
2717
2718	void HELPER(sve_sqaddi_b)(void d, void a, int32_t b, uint32_t desc)
2719	{
2720	intptr_t i, oprsz = simd_oprsz(desc);
2721
2722	for (i = 0; i < oprsz; i += sizeof(int8_t)) {
2723	(int8_t )(d + i) = DO_SQADD_B(b, (int8_t )(a + i));
2724	}
2725	}
2726
2727	void HELPER(sve_sqaddi_h)(void d, void a, int32_t b, uint32_t desc)
2728	{
2729	intptr_t i, oprsz = simd_oprsz(desc);
2730
2731	for (i = 0; i < oprsz; i += sizeof(int16_t)) {
2732	(int16_t )(d + i) = DO_SQADD_H(b, (int16_t )(a + i));
2733	}
2734	}
2735
2736	void HELPER(sve_sqaddi_s)(void d, void a, int64_t b, uint32_t desc)
2737	{
2738	intptr_t i, oprsz = simd_oprsz(desc);
2739
2740	for (i = 0; i < oprsz; i += sizeof(int32_t)) {
2741	(int32_t )(d + i) = DO_SQADD_S(b, (int32_t )(a + i));
2742	}
2743	}
2744
2745	void HELPER(sve_sqaddi_d)(void d, void a, int64_t b, uint32_t desc)
2746	{
2747	intptr_t i, oprsz = simd_oprsz(desc);
2748
2749	for (i = 0; i < oprsz; i += sizeof(int64_t)) {
2750	(int64_t )(d + i) = do_sqadd_d(b, (int64_t )(a + i));
2751	}
2752	}
2753
2754	/*
2755	* Unsigned saturating addition with scalar operand.
2756	*/
2757
2758	void HELPER(sve_uqaddi_b)(void d, void a, int32_t b, uint32_t desc)
2759	{
2760	intptr_t i, oprsz = simd_oprsz(desc);
2761
2762	for (i = 0; i < oprsz; i += sizeof(uint8_t)) {
2763	(uint8_t )(d + i) = DO_UQADD_B(b, (uint8_t )(a + i));
2764	}
2765	}
2766
2767	void HELPER(sve_uqaddi_h)(void d, void a, int32_t b, uint32_t desc)
2768	{
2769	intptr_t i, oprsz = simd_oprsz(desc);
2770
2771	for (i = 0; i < oprsz; i += sizeof(uint16_t)) {
2772	(uint16_t )(d + i) = DO_UQADD_H(b, (uint16_t )(a + i));
2773	}
2774	}
2775
2776	void HELPER(sve_uqaddi_s)(void d, void a, int64_t b, uint32_t desc)
2777	{
2778	intptr_t i, oprsz = simd_oprsz(desc);
2779
2780	for (i = 0; i < oprsz; i += sizeof(uint32_t)) {
2781	(uint32_t )(d + i) = DO_UQADD_S(b, (uint32_t )(a + i));
2782	}
2783	}
2784
2785	void HELPER(sve_uqaddi_d)(void d, void a, uint64_t b, uint32_t desc)
2786	{
2787	intptr_t i, oprsz = simd_oprsz(desc);
2788
2789	for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
2790	(uint64_t )(d + i) = do_uqadd_d(b, (uint64_t )(a + i));
2791	}
2792	}
2793
2794	void HELPER(sve_uqsubi_d)(void d, void a, uint64_t b, uint32_t desc)
2795	{
2796	intptr_t i, oprsz = simd_oprsz(desc);
2797
2798	for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
2799	(uint64_t )(d + i) = do_uqsub_d((uint64_t )(a + i), b);
2800	}
2801	}
2802
2803	/* Two operand predicated copy immediate with merge. All valid immediates
2804	* can fit within 17 signed bits in the simd_data field.
2805	*/
2806	void HELPER(sve_cpy_m_b)(void vd, void vn, void *vg,
2807	uint64_t mm, uint32_t desc)
2808	{
2809	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2810	uint64_t d = vd, n = vn;
2811	uint8_t *pg = vg;
2812
2813	mm = dup_const(MO_8, mm);
2814	for (i = 0; i < opr_sz; i += 1) {
2815	uint64_t nn = n[i];
2816	uint64_t pp = expand_pred_b(pg[H1(i)]);
2817	d[i] = (mm & pp) \| (nn & ~pp);
2818	}
2819	}
2820
2821	void HELPER(sve_cpy_m_h)(void vd, void vn, void *vg,
2822	uint64_t mm, uint32_t desc)
2823	{
2824	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2825	uint64_t d = vd, n = vn;
2826	uint8_t *pg = vg;
2827
2828	mm = dup_const(MO_16, mm);
2829	for (i = 0; i < opr_sz; i += 1) {
2830	uint64_t nn = n[i];
2831	uint64_t pp = expand_pred_h(pg[H1(i)]);
2832	d[i] = (mm & pp) \| (nn & ~pp);
2833	}
2834	}
2835
2836	void HELPER(sve_cpy_m_s)(void vd, void vn, void *vg,
2837	uint64_t mm, uint32_t desc)
2838	{
2839	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2840	uint64_t d = vd, n = vn;
2841	uint8_t *pg = vg;
2842
2843	mm = dup_const(MO_32, mm);
2844	for (i = 0; i < opr_sz; i += 1) {
2845	uint64_t nn = n[i];
2846	uint64_t pp = expand_pred_s(pg[H1(i)]);
2847	d[i] = (mm & pp) \| (nn & ~pp);
2848	}
2849	}
2850
2851	void HELPER(sve_cpy_m_d)(void vd, void vn, void *vg,
2852	uint64_t mm, uint32_t desc)
2853	{
2854	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2855	uint64_t d = vd, n = vn;
2856	uint8_t *pg = vg;
2857
2858	for (i = 0; i < opr_sz; i += 1) {
2859	uint64_t nn = n[i];
2860	d[i] = (pg[H1(i)] & 1 ? mm : nn);
2861	}
2862	}
2863
2864	void HELPER(sve_cpy_z_b)(void vd, void vg, uint64_t val, uint32_t desc)
2865	{
2866	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2867	uint64_t *d = vd;
2868	uint8_t *pg = vg;
2869
2870	val = dup_const(MO_8, val);
2871	for (i = 0; i < opr_sz; i += 1) {
2872	d[i] = val & expand_pred_b(pg[H1(i)]);
2873	}
2874	}
2875
2876	void HELPER(sve_cpy_z_h)(void vd, void vg, uint64_t val, uint32_t desc)
2877	{
2878	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2879	uint64_t *d = vd;
2880	uint8_t *pg = vg;
2881
2882	val = dup_const(MO_16, val);
2883	for (i = 0; i < opr_sz; i += 1) {
2884	d[i] = val & expand_pred_h(pg[H1(i)]);
2885	}
2886	}
2887
2888	void HELPER(sve_cpy_z_s)(void vd, void vg, uint64_t val, uint32_t desc)
2889	{
2890	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2891	uint64_t *d = vd;
2892	uint8_t *pg = vg;
2893
2894	val = dup_const(MO_32, val);
2895	for (i = 0; i < opr_sz; i += 1) {
2896	d[i] = val & expand_pred_s(pg[H1(i)]);
2897	}
2898	}
2899
2900	void HELPER(sve_cpy_z_d)(void vd, void vg, uint64_t val, uint32_t desc)
2901	{
2902	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2903	uint64_t *d = vd;
2904	uint8_t *pg = vg;
2905
2906	for (i = 0; i < opr_sz; i += 1) {
2907	d[i] = (pg[H1(i)] & 1 ? val : 0);
2908	}
2909	}
2910
2911	/* Big-endian hosts need to frob the byte indices. If the copy
2912	* happens to be 8-byte aligned, then no frobbing necessary.
2913	*/
2914	static void swap_memmove(void vd, void vs, size_t n)
2915	{
2916	uintptr_t d = (uintptr_t)vd;
2917	uintptr_t s = (uintptr_t)vs;
2918	uintptr_t o = (d \| s \| n) & 7;
2919	size_t i;
2920
2921	#if !HOST_BIG_ENDIAN
2922	o = 0;
2923	#endif
2924	switch (o) {
2925	case 0:
2926	memmove(vd, vs, n);
2927	break;
2928
2929	case 4:
2930	if (d < s \|\| d >= s + n) {
2931	for (i = 0; i < n; i += 4) {
2932	(uint32_t )H1_4(d + i) = (uint32_t )H1_4(s + i);
2933	}
2934	} else {
2935	for (i = n; i > 0; ) {
2936	i -= 4;
2937	(uint32_t )H1_4(d + i) = (uint32_t )H1_4(s + i);
2938	}
2939	}
2940	break;
2941
2942	case 2:
2943	case 6:
2944	if (d < s \|\| d >= s + n) {
2945	for (i = 0; i < n; i += 2) {
2946	(uint16_t )H1_2(d + i) = (uint16_t )H1_2(s + i);
2947	}
2948	} else {
2949	for (i = n; i > 0; ) {
2950	i -= 2;
2951	(uint16_t )H1_2(d + i) = (uint16_t )H1_2(s + i);
2952	}
2953	}
2954	break;
2955
2956	default:
2957	if (d < s \|\| d >= s + n) {
2958	for (i = 0; i < n; i++) {
2959	(uint8_t )H1(d + i) = (uint8_t )H1(s + i);
2960	}
2961	} else {
2962	for (i = n; i > 0; ) {
2963	i -= 1;
2964	(uint8_t )H1(d + i) = (uint8_t )H1(s + i);
2965	}
2966	}
2967	break;
2968	}
2969	}
2970
2971	/* Similarly for memset of 0. */
2972	static void swap_memzero(void *vd, size_t n)
2973	{
2974	uintptr_t d = (uintptr_t)vd;
2975	uintptr_t o = (d \| n) & 7;
2976	size_t i;
2977
2978	/* Usually, the first bit of a predicate is set, so N is 0. */
2979	if (likely(n == 0)) {
2980	return;
2981	}
2982
2983	#if !HOST_BIG_ENDIAN
2984	o = 0;
2985	#endif
2986	switch (o) {
2987	case 0:
2988	memset(vd, 0, n);
2989	break;
2990
2991	case 4:
2992	for (i = 0; i < n; i += 4) {
2993	(uint32_t )H1_4(d + i) = 0;
2994	}
2995	break;
2996
2997	case 2:
2998	case 6:
2999	for (i = 0; i < n; i += 2) {
3000	(uint16_t )H1_2(d + i) = 0;
3001	}
3002	break;
3003
3004	default:
3005	for (i = 0; i < n; i++) {
3006	(uint8_t )H1(d + i) = 0;
3007	}
3008	break;
3009	}
3010	}
3011
3012	void HELPER(sve_ext)(void vd, void vn, void *vm, uint32_t desc)
3013	{
3014	intptr_t opr_sz = simd_oprsz(desc);
3015	size_t n_ofs = simd_data(desc);
3016	size_t n_siz = opr_sz - n_ofs;
3017
3018	if (vd != vm) {
3019	swap_memmove(vd, vn + n_ofs, n_siz);
3020	swap_memmove(vd + n_siz, vm, n_ofs);
3021	} else if (vd != vn) {
3022	swap_memmove(vd + n_siz, vd, n_ofs);
3023	swap_memmove(vd, vn + n_ofs, n_siz);
3024	} else {
3025	/* vd == vn == vm. Need temp space. */
3026	ARMVectorReg tmp;
3027	swap_memmove(&tmp, vm, n_ofs);
3028	swap_memmove(vd, vd + n_ofs, n_siz);
3029	memcpy(vd + n_siz, &tmp, n_ofs);
3030	}
3031	}
3032
3033	#define DO_INSR(NAME, TYPE, H) \
3034	void HELPER(NAME)(void vd, void vn, uint64_t val, uint32_t desc) \
3035	{ \
3036	intptr_t opr_sz = simd_oprsz(desc); \
3037	swap_memmove(vd + sizeof(TYPE), vn, opr_sz - sizeof(TYPE)); \
3038	(TYPE )(vd + H(0)) = val; \
3039	}
3040
3041	DO_INSR(sve_insr_b, uint8_t, H1)
3042	DO_INSR(sve_insr_h, uint16_t, H1_2)
3043	DO_INSR(sve_insr_s, uint32_t, H1_4)
3044	DO_INSR(sve_insr_d, uint64_t, H1_8)
3045
3046	#undef DO_INSR
3047
3048	void HELPER(sve_rev_b)(void vd, void vn, uint32_t desc)
3049	{
3050	intptr_t i, j, opr_sz = simd_oprsz(desc);
3051	for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
3052	uint64_t f = (uint64_t )(vn + i);
3053	uint64_t b = (uint64_t )(vn + j);
3054	(uint64_t )(vd + i) = bswap64(b);
3055	(uint64_t )(vd + j) = bswap64(f);
3056	}
3057	}
3058
3059	void HELPER(sve_rev_h)(void vd, void vn, uint32_t desc)
3060	{
3061	intptr_t i, j, opr_sz = simd_oprsz(desc);
3062	for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
3063	uint64_t f = (uint64_t )(vn + i);
3064	uint64_t b = (uint64_t )(vn + j);
3065	(uint64_t )(vd + i) = hswap64(b);
3066	(uint64_t )(vd + j) = hswap64(f);
3067	}
3068	}
3069
3070	void HELPER(sve_rev_s)(void vd, void vn, uint32_t desc)
3071	{
3072	intptr_t i, j, opr_sz = simd_oprsz(desc);
3073	for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
3074	uint64_t f = (uint64_t )(vn + i);
3075	uint64_t b = (uint64_t )(vn + j);
3076	(uint64_t )(vd + i) = rol64(b, 32);
3077	(uint64_t )(vd + j) = rol64(f, 32);
3078	}
3079	}
3080
3081	void HELPER(sve_rev_d)(void vd, void vn, uint32_t desc)
3082	{
3083	intptr_t i, j, opr_sz = simd_oprsz(desc);
3084	for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
3085	uint64_t f = (uint64_t )(vn + i);
3086	uint64_t b = (uint64_t )(vn + j);
3087	(uint64_t )(vd + i) = b;
3088	(uint64_t )(vd + j) = f;
3089	}
3090	}
3091
3092	/*
3093	* TODO: This could use half_shuffle64 and similar bit tricks to
3094	* expand blocks of bits at once.
3095	*/
3096	#define DO_PMOV_PV(NAME, ESIZE) \
3097	void HELPER(NAME)(void vd, void vs, uint32_t desc) \
3098	{ \
3099	unsigned vl = simd_oprsz(desc); \
3100	unsigned idx = simd_data(desc); \
3101	unsigned elements = vl / ESIZE; \
3102	ARMPredicateReg *d = vd; \
3103	ARMVectorReg *s = vs; \
3104	memset(d, 0, sizeof(*d)); \
3105	for (unsigned e = 0; e < elements; ++e) { \
3106	depositn(d->p, e * ESIZE, 1, extractn(s->d, elements * idx + e, 1)); \
3107	} \
3108	}
3109
3110	DO_PMOV_PV(pmov_pv_h, 2)
3111	DO_PMOV_PV(pmov_pv_s, 4)
3112	DO_PMOV_PV(pmov_pv_d, 8)
3113
3114	#undef DO_PMOV_PV
3115
3116	/*
3117	* TODO: This could use half_unshuffle64 and similar bit tricks to
3118	* compress blocks of bits at once.
3119	*/
3120	#define DO_PMOV_VP(NAME, ESIZE) \
3121	void HELPER(NAME)(void vd, void vs, uint32_t desc) \
3122	{ \
3123	unsigned vl = simd_oprsz(desc); \
3124	unsigned idx = simd_data(desc); \
3125	unsigned elements = vl / ESIZE; \
3126	ARMVectorReg *d = vd; \
3127	ARMPredicateReg *s = vs; \
3128	if (idx == 0) { \
3129	memset(d, 0, vl); \
3130	} \
3131	for (unsigned e = 0; e < elements; ++e) { \
3132	depositn(d->d, elements * idx + e, 1, extractn(s->p, e * ESIZE, 1)); \
3133	} \
3134	}
3135
3136	DO_PMOV_VP(pmov_vp_h, 2)
3137	DO_PMOV_VP(pmov_vp_s, 4)
3138	DO_PMOV_VP(pmov_vp_d, 8)
3139
3140	#undef DO_PMOV_VP
3141
3142	typedef void tb_impl_fn(void , void , void , void , uintptr_t, bool);
3143
3144	static inline void do_tbl1(void vd, void vn, void *vm, uint32_t desc,
3145	bool is_tbx, tb_impl_fn *fn)
3146	{
3147	ARMVectorReg scratch;
3148	uintptr_t oprsz = simd_oprsz(desc);
3149
3150	if (unlikely(vd == vn)) {
3151	vn = memcpy(&scratch, vn, oprsz);
3152	}
3153
3154	fn(vd, vn, NULL, vm, oprsz, is_tbx);
3155	}
3156
3157	static inline void do_tbl2(void vd, void vn0, void vn1, void vm,
3158	uint32_t desc, bool is_tbx, tb_impl_fn *fn)
3159	{
3160	ARMVectorReg scratch;
3161	uintptr_t oprsz = simd_oprsz(desc);
3162
3163	if (unlikely(vd == vn0)) {
3164	vn0 = memcpy(&scratch, vn0, oprsz);
3165	if (vd == vn1) {
3166	vn1 = vn0;
3167	}
3168	} else if (unlikely(vd == vn1)) {
3169	vn1 = memcpy(&scratch, vn1, oprsz);
3170	}
3171
3172	fn(vd, vn0, vn1, vm, oprsz, is_tbx);
3173	}
3174
3175	#define DO_TB(SUFF, TYPE, H) \
3176	static inline void do_tb_##SUFF(void vd, void vt0, void *vt1, \
3177	void *vm, uintptr_t oprsz, bool is_tbx) \
3178	{ \
3179	TYPE d = vd, tbl0 = vt0, tbl1 = vt1, indexes = vm; \
3180	uintptr_t i, nelem = oprsz / sizeof(TYPE); \
3181	for (i = 0; i < nelem; ++i) { \
3182	TYPE index = indexes[H1(i)], val = 0; \
3183	if (index < nelem) { \
3184	val = tbl0[H(index)]; \
3185	} else { \
3186	index -= nelem; \
3187	if (tbl1 && index < nelem) { \
3188	val = tbl1[H(index)]; \
3189	} else if (is_tbx) { \
3190	continue; \
3191	} \
3192	} \
3193	d[H(i)] = val; \
3194	} \
3195	} \
3196	void HELPER(sve_tbl_##SUFF)(void vd, void vn, void *vm, uint32_t desc) \
3197	{ \
3198	do_tbl1(vd, vn, vm, desc, false, do_tb_##SUFF); \
3199	} \
3200	void HELPER(sve2_tbl_##SUFF)(void vd, void vn0, void *vn1, \
3201	void *vm, uint32_t desc) \
3202	{ \
3203	do_tbl2(vd, vn0, vn1, vm, desc, false, do_tb_##SUFF); \
3204	} \
3205	void HELPER(sve2_tbx_##SUFF)(void vd, void vn, void *vm, uint32_t desc) \
3206	{ \
3207	do_tbl1(vd, vn, vm, desc, true, do_tb_##SUFF); \
3208	}
3209
3210	DO_TB(b, uint8_t, H1)
3211	DO_TB(h, uint16_t, H2)
3212	DO_TB(s, uint32_t, H4)
3213	DO_TB(d, uint64_t, H8)
3214
3215	#undef DO_TB
3216
3217	#define DO_UNPK(NAME, TYPED, TYPES, HD, HS) \
3218	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
3219	{ \
3220	intptr_t i, opr_sz = simd_oprsz(desc); \
3221	TYPED *d = vd; \
3222	TYPES *n = vn; \
3223	ARMVectorReg tmp; \
3224	if (unlikely(vn - vd < opr_sz)) { \
3225	n = memcpy(&tmp, n, opr_sz / 2); \
3226	} \
3227	for (i = 0; i < opr_sz / sizeof(TYPED); i++) { \
3228	d[HD(i)] = n[HS(i)]; \
3229	} \
3230	}
3231
3232	DO_UNPK(sve_sunpk_h, int16_t, int8_t, H2, H1)
3233	DO_UNPK(sve_sunpk_s, int32_t, int16_t, H4, H2)
3234	DO_UNPK(sve_sunpk_d, int64_t, int32_t, H8, H4)
3235
3236	DO_UNPK(sve_uunpk_h, uint16_t, uint8_t, H2, H1)
3237	DO_UNPK(sve_uunpk_s, uint32_t, uint16_t, H4, H2)
3238	DO_UNPK(sve_uunpk_d, uint64_t, uint32_t, H8, H4)
3239
3240	#undef DO_UNPK
3241
3242	/* Mask of bits included in the even numbered predicates of width esz.
3243	* We also use this for expand_bits/compress_bits, and so extend the
3244	* same pattern out to 16-bit units.
3245	*/
3246	static const uint64_t even_bit_esz_masks[5] = {
3247	0x5555555555555555ull,
3248	0x3333333333333333ull,
3249	0x0f0f0f0f0f0f0f0full,
3250	0x00ff00ff00ff00ffull,
3251	0x0000ffff0000ffffull,
3252	};
3253
3254	/* Zero-extend units of 2N bits to units of 2(N+1) bits.
3255	* For N==0, this corresponds to the operation that in qemu/bitops.h
3256	* we call half_shuffle64; this algorithm is from Hacker's Delight,
3257	* section 7-2 Shuffling Bits.
3258	*/
3259	static uint64_t expand_bits(uint64_t x, int n)
3260	{
3261	int i;
3262
3263	x &= 0xffffffffu;
3264	for (i = 4; i >= n; i--) {
3265	int sh = 1 << i;
3266	x = ((x << sh) \| x) & even_bit_esz_masks[i];
3267	}
3268	return x;
3269	}
3270
3271	/* Compress units of 2(N+1) bits to units of 2N bits.
3272	* For N==0, this corresponds to the operation that in qemu/bitops.h
3273	* we call half_unshuffle64; this algorithm is from Hacker's Delight,
3274	* section 7-2 Shuffling Bits, where it is called an inverse half shuffle.
3275	*/
3276	static uint64_t compress_bits(uint64_t x, int n)
3277	{
3278	int i;
3279
3280	for (i = n; i <= 4; i++) {
3281	int sh = 1 << i;
3282	x &= even_bit_esz_masks[i];
3283	x = (x >> sh) \| x;
3284	}
3285	return x & 0xffffffffu;
3286	}
3287
3288	void HELPER(sve_zip_p)(void vd, void vn, void *vm, uint32_t pred_desc)
3289	{
3290	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
3291	int esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
3292	intptr_t high = FIELD_EX32(pred_desc, PREDDESC, DATA);
3293	int esize = 1 << esz;
3294	uint64_t *d = vd;
3295	intptr_t i;
3296
3297	if (oprsz <= 8) {
3298	uint64_t nn = (uint64_t )vn;
3299	uint64_t mm = (uint64_t )vm;
3300	int half = 4 * oprsz;
3301
3302	nn = extract64(nn, high * half, half);
3303	mm = extract64(mm, high * half, half);
3304	nn = expand_bits(nn, esz);
3305	mm = expand_bits(mm, esz);
3306	d[0] = nn \| (mm << esize);
3307	} else {
3308	ARMPredicateReg tmp;
3309
3310	/* We produce output faster than we consume input.
3311	Therefore we must be mindful of possible overlap. */
3312	if (vd == vn) {
3313	vn = memcpy(&tmp, vn, oprsz);
3314	if (vd == vm) {
3315	vm = vn;
3316	}
3317	} else if (vd == vm) {
3318	vm = memcpy(&tmp, vm, oprsz);
3319	}
3320	if (high) {
3321	high = oprsz >> 1;
3322	}
3323
3324	if ((oprsz & 7) == 0) {
3325	uint32_t n = vn, m = vm;
3326	high >>= 2;
3327
3328	for (i = 0; i < oprsz / 8; i++) {
3329	uint64_t nn = n[H4(high + i)];
3330	uint64_t mm = m[H4(high + i)];
3331
3332	nn = expand_bits(nn, esz);
3333	mm = expand_bits(mm, esz);
3334	d[i] = nn \| (mm << esize);
3335	}
3336	} else {
3337	uint8_t n = vn, m = vm;
3338	uint16_t *d16 = vd;
3339
3340	for (i = 0; i < oprsz / 2; i++) {
3341	uint16_t nn = n[H1(high + i)];
3342	uint16_t mm = m[H1(high + i)];
3343
3344	nn = expand_bits(nn, esz);
3345	mm = expand_bits(mm, esz);
3346	d16[H2(i)] = nn \| (mm << esize);
3347	}
3348	}
3349	}
3350	}
3351
3352	void HELPER(sve_uzp_p)(void vd, void vn, void *vm, uint32_t pred_desc)
3353	{
3354	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
3355	int esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
3356	int odd = FIELD_EX32(pred_desc, PREDDESC, DATA) << esz;
3357	uint64_t d = vd, n = vn, *m = vm;
3358	uint64_t l, h;
3359	intptr_t i;
3360
3361	if (oprsz <= 8) {
3362	l = compress_bits(n[0] >> odd, esz);
3363	h = compress_bits(m[0] >> odd, esz);
3364	d[0] = l \| (h << (4 * oprsz));
3365	} else {
3366	ARMPredicateReg tmp_m;
3367	intptr_t oprsz_16 = oprsz / 16;
3368
3369	if ((vm - vd) < (uintptr_t)oprsz) {
3370	m = memcpy(&tmp_m, vm, oprsz);
3371	}
3372
3373	for (i = 0; i < oprsz_16; i++) {
3374	l = n[2 * i + 0];
3375	h = n[2 * i + 1];
3376	l = compress_bits(l >> odd, esz);
3377	h = compress_bits(h >> odd, esz);
3378	d[i] = l \| (h << 32);
3379	}
3380
3381	/*
3382	* For VL which is not a multiple of 512, the results from M do not
3383	* align nicely with the uint64_t for D. Put the aligned results
3384	* from M into TMP_M and then copy it into place afterward.
3385	*/
3386	if (oprsz & 15) {
3387	int final_shift = (oprsz & 15) * 2;
3388
3389	l = n[2 * i + 0];
3390	h = n[2 * i + 1];
3391	l = compress_bits(l >> odd, esz);
3392	h = compress_bits(h >> odd, esz);
3393	d[i] = l \| (h << final_shift);
3394
3395	for (i = 0; i < oprsz_16; i++) {
3396	l = m[2 * i + 0];
3397	h = m[2 * i + 1];
3398	l = compress_bits(l >> odd, esz);
3399	h = compress_bits(h >> odd, esz);
3400	tmp_m.p[i] = l \| (h << 32);
3401	}
3402	l = m[2 * i + 0];
3403	h = m[2 * i + 1];
3404	l = compress_bits(l >> odd, esz);
3405	h = compress_bits(h >> odd, esz);
3406	tmp_m.p[i] = l \| (h << final_shift);
3407
3408	swap_memmove(vd + oprsz / 2, &tmp_m, oprsz / 2);
3409	} else {
3410	for (i = 0; i < oprsz_16; i++) {
3411	l = m[2 * i + 0];
3412	h = m[2 * i + 1];
3413	l = compress_bits(l >> odd, esz);
3414	h = compress_bits(h >> odd, esz);
3415	d[oprsz_16 + i] = l \| (h << 32);
3416	}
3417	}
3418	}
3419	}
3420
3421	void HELPER(sve_trn_p)(void vd, void vn, void *vm, uint32_t pred_desc)
3422	{
3423	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
3424	int esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
3425	int odd = FIELD_EX32(pred_desc, PREDDESC, DATA);
3426	uint64_t d = vd, n = vn, *m = vm;
3427	uint64_t mask;
3428	int shr, shl;
3429	intptr_t i;
3430
3431	shl = 1 << esz;
3432	shr = 0;
3433	mask = even_bit_esz_masks[esz];
3434	if (odd) {
3435	mask <<= shl;
3436	shr = shl;
3437	shl = 0;
3438	}
3439
3440	for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
3441	uint64_t nn = (n[i] & mask) >> shr;
3442	uint64_t mm = (m[i] & mask) << shl;
3443	d[i] = nn + mm;
3444	}
3445	}
3446
3447	/* Reverse units of 2*N bits. /
3448	static uint64_t reverse_bits_64(uint64_t x, int n)
3449	{
3450	int i, sh;
3451
3452	x = bswap64(x);
3453	for (i = 2, sh = 4; i >= n; i--, sh >>= 1) {
3454	uint64_t mask = even_bit_esz_masks[i];
3455	x = ((x & mask) << sh) \| ((x >> sh) & mask);
3456	}
3457	return x;
3458	}
3459
3460	static uint8_t reverse_bits_8(uint8_t x, int n)
3461	{
3462	static const uint8_t mask[3] = { 0x55, 0x33, 0x0f };
3463	int i, sh;
3464
3465	for (i = 2, sh = 4; i >= n; i--, sh >>= 1) {
3466	x = ((x & mask[i]) << sh) \| ((x >> sh) & mask[i]);
3467	}
3468	return x;
3469	}
3470
3471	void HELPER(sve_rev_p)(void vd, void vn, uint32_t pred_desc)
3472	{
3473	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
3474	int esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
3475	intptr_t i, oprsz_2 = oprsz / 2;
3476
3477	if (oprsz <= 8) {
3478	uint64_t l = (uint64_t )vn;
3479	l = reverse_bits_64(l << (64 - 8 * oprsz), esz);
3480	(uint64_t )vd = l;
3481	} else if ((oprsz & 15) == 0) {
3482	for (i = 0; i < oprsz_2; i += 8) {
3483	intptr_t ih = oprsz - 8 - i;
3484	uint64_t l = reverse_bits_64((uint64_t )(vn + i), esz);
3485	uint64_t h = reverse_bits_64((uint64_t )(vn + ih), esz);
3486	(uint64_t )(vd + i) = h;
3487	(uint64_t )(vd + ih) = l;
3488	}
3489	} else {
3490	for (i = 0; i < oprsz_2; i += 1) {
3491	intptr_t il = H1(i);
3492	intptr_t ih = H1(oprsz - 1 - i);
3493	uint8_t l = reverse_bits_8((uint8_t )(vn + il), esz);
3494	uint8_t h = reverse_bits_8((uint8_t )(vn + ih), esz);
3495	(uint8_t )(vd + il) = h;
3496	(uint8_t )(vd + ih) = l;
3497	}
3498	}
3499	}
3500
3501	void HELPER(sve_punpk_p)(void vd, void vn, uint32_t pred_desc)
3502	{
3503	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
3504	intptr_t high = FIELD_EX32(pred_desc, PREDDESC, DATA);
3505	uint64_t *d = vd;
3506	intptr_t i;
3507
3508	if (oprsz <= 8) {
3509	uint64_t nn = (uint64_t )vn;
3510	int half = 4 * oprsz;
3511
3512	nn = extract64(nn, high * half, half);
3513	nn = expand_bits(nn, 0);
3514	d[0] = nn;
3515	} else {
3516	ARMPredicateReg tmp_n;
3517
3518	/* We produce output faster than we consume input.
3519	Therefore we must be mindful of possible overlap. */
3520	if ((vn - vd) < (uintptr_t)oprsz) {
3521	vn = memcpy(&tmp_n, vn, oprsz);
3522	}
3523	if (high) {
3524	high = oprsz >> 1;
3525	}
3526
3527	if ((oprsz & 7) == 0) {
3528	uint32_t *n = vn;
3529	high >>= 2;
3530
3531	for (i = 0; i < oprsz / 8; i++) {
3532	uint64_t nn = n[H4(high + i)];
3533	d[i] = expand_bits(nn, 0);
3534	}
3535	} else {
3536	uint16_t *d16 = vd;
3537	uint8_t *n = vn;
3538
3539	for (i = 0; i < oprsz / 2; i++) {
3540	uint16_t nn = n[H1(high + i)];
3541	d16[H2(i)] = expand_bits(nn, 0);
3542	}
3543	}
3544	}
3545	}
3546
3547	#define DO_ZIP(NAME, TYPE, H) \
3548	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
3549	{ \
3550	intptr_t oprsz = simd_oprsz(desc); \
3551	intptr_t odd_ofs = simd_data(desc); \
3552	intptr_t i, oprsz_2 = oprsz / 2; \
3553	ARMVectorReg tmp_n, tmp_m; \
3554	/* We produce output faster than we consume input. \
3555	Therefore we must be mindful of possible overlap. */ \
3556	if (unlikely((vn - vd) < (uintptr_t)oprsz)) { \
3557	vn = memcpy(&tmp_n, vn, oprsz); \
3558	} \
3559	if (unlikely((vm - vd) < (uintptr_t)oprsz)) { \
3560	vm = memcpy(&tmp_m, vm, oprsz); \
3561	} \
3562	for (i = 0; i < oprsz_2; i += sizeof(TYPE)) { \
3563	(TYPE )(vd + H(2 * i + 0)) = (TYPE )(vn + odd_ofs + H(i)); \
3564	(TYPE )(vd + H(2 * i + sizeof(TYPE))) = \
3565	(TYPE )(vm + odd_ofs + H(i)); \
3566	} \
3567	if (sizeof(TYPE) == 16 && unlikely(oprsz & 16)) { \
3568	memset(vd + oprsz - 16, 0, 16); \
3569	} \
3570	}
3571
3572	DO_ZIP(sve_zip_b, uint8_t, H1)
3573	DO_ZIP(sve_zip_h, uint16_t, H1_2)
3574	DO_ZIP(sve_zip_s, uint32_t, H1_4)
3575	DO_ZIP(sve_zip_d, uint64_t, H1_8)
3576	DO_ZIP(sve2_zip_q, Int128, )
3577
3578	#define DO_UZP(NAME, TYPE, H) \
3579	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
3580	{ \
3581	intptr_t oprsz = simd_oprsz(desc); \
3582	intptr_t odd_ofs = simd_data(desc); \
3583	intptr_t i, p; \
3584	ARMVectorReg tmp_m; \
3585	if (unlikely((vm - vd) < (uintptr_t)oprsz)) { \
3586	vm = memcpy(&tmp_m, vm, oprsz); \
3587	} \
3588	i = 0, p = odd_ofs; \
3589	do { \
3590	(TYPE )(vd + H(i)) = (TYPE )(vn + H(p)); \
3591	i += sizeof(TYPE), p += 2 * sizeof(TYPE); \
3592	} while (p < oprsz); \
3593	p -= oprsz; \
3594	do { \
3595	(TYPE )(vd + H(i)) = (TYPE )(vm + H(p)); \
3596	i += sizeof(TYPE), p += 2 * sizeof(TYPE); \
3597	} while (p < oprsz); \
3598	tcg_debug_assert(i == oprsz); \
3599	}
3600
3601	DO_UZP(sve_uzp_b, uint8_t, H1)
3602	DO_UZP(sve_uzp_h, uint16_t, H1_2)
3603	DO_UZP(sve_uzp_s, uint32_t, H1_4)
3604	DO_UZP(sve_uzp_d, uint64_t, H1_8)
3605	DO_UZP(sve2_uzp_q, Int128, )
3606
3607	typedef void perseg_zzz_fn(void vd, void vn, void *vm, uint32_t desc);
3608
3609	static void do_perseg_zzz(void vd, void vn, void *vm,
3610	uint32_t desc, perseg_zzz_fn *fn)
3611	{
3612	intptr_t oprsz = simd_oprsz(desc);
3613
3614	desc = simd_desc(16, 16, simd_data(desc));
3615	for (intptr_t i = 0; i < oprsz; i += 16) {
3616	fn(vd + i, vn + i, vm + i, desc);
3617	}
3618	}
3619
3620	#define DO_PERSEG_ZZZ(NAME, FUNC) \
3621	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
3622	{ do_perseg_zzz(vd, vn, vm, desc, FUNC); }
3623
3624	DO_PERSEG_ZZZ(sve2p1_uzpq_b, helper_sve_uzp_b)
3625	DO_PERSEG_ZZZ(sve2p1_uzpq_h, helper_sve_uzp_h)
3626	DO_PERSEG_ZZZ(sve2p1_uzpq_s, helper_sve_uzp_s)
3627	DO_PERSEG_ZZZ(sve2p1_uzpq_d, helper_sve_uzp_d)
3628
3629	DO_PERSEG_ZZZ(sve2p1_zipq_b, helper_sve_zip_b)
3630	DO_PERSEG_ZZZ(sve2p1_zipq_h, helper_sve_zip_h)
3631	DO_PERSEG_ZZZ(sve2p1_zipq_s, helper_sve_zip_s)
3632	DO_PERSEG_ZZZ(sve2p1_zipq_d, helper_sve_zip_d)
3633
3634	DO_PERSEG_ZZZ(sve2p1_tblq_b, helper_sve_tbl_b)
3635	DO_PERSEG_ZZZ(sve2p1_tblq_h, helper_sve_tbl_h)
3636	DO_PERSEG_ZZZ(sve2p1_tblq_s, helper_sve_tbl_s)
3637	DO_PERSEG_ZZZ(sve2p1_tblq_d, helper_sve_tbl_d)
3638
3639	DO_PERSEG_ZZZ(sve2p1_tbxq_b, helper_sve2_tbx_b)
3640	DO_PERSEG_ZZZ(sve2p1_tbxq_h, helper_sve2_tbx_h)
3641	DO_PERSEG_ZZZ(sve2p1_tbxq_s, helper_sve2_tbx_s)
3642	DO_PERSEG_ZZZ(sve2p1_tbxq_d, helper_sve2_tbx_d)
3643
3644	#undef DO_PERSEG_ZZZ
3645
3646	#define DO_TRN(NAME, TYPE, H) \
3647	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
3648	{ \
3649	intptr_t oprsz = simd_oprsz(desc); \
3650	intptr_t odd_ofs = simd_data(desc); \
3651	intptr_t i; \
3652	for (i = 0; i < oprsz; i += 2 * sizeof(TYPE)) { \
3653	TYPE ae = (TYPE )(vn + H(i + odd_ofs)); \
3654	TYPE be = (TYPE )(vm + H(i + odd_ofs)); \
3655	(TYPE )(vd + H(i + 0)) = ae; \
3656	(TYPE )(vd + H(i + sizeof(TYPE))) = be; \
3657	} \
3658	if (sizeof(TYPE) == 16 && unlikely(oprsz & 16)) { \
3659	memset(vd + oprsz - 16, 0, 16); \
3660	} \
3661	}
3662
3663	DO_TRN(sve_trn_b, uint8_t, H1)
3664	DO_TRN(sve_trn_h, uint16_t, H1_2)
3665	DO_TRN(sve_trn_s, uint32_t, H1_4)
3666	DO_TRN(sve_trn_d, uint64_t, H1_8)
3667	DO_TRN(sve2_trn_q, Int128, )
3668
3669	#undef DO_ZIP
3670	#undef DO_UZP
3671	#undef DO_TRN
3672
3673	#define DO_COMPACT(NAME, TYPE, H) \
3674	void HELPER(NAME)(void vd, void vn, void *vg, uint32_t desc) \
3675	{ \
3676	intptr_t j = 0, oprsz = simd_oprsz(desc); \
3677	for (intptr_t i = 0; i < oprsz; ) { \
3678	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
3679	do { \
3680	if (pg & 1) { \
3681	(TYPE )(vd + H(j)) = (TYPE )(vn + H(i)); \
3682	j += sizeof(TYPE); \
3683	} \
3684	i += sizeof(TYPE); \
3685	pg >>= sizeof(TYPE); \
3686	} while (i & 15); \
3687	} \
3688	for (; j < oprsz; j += sizeof(TYPE)) { \
3689	(TYPE )(vd + H(j)) = 0; \
3690	} \
3691	}
3692
3693	DO_COMPACT(sve_compact_b, uint8_t, H1)
3694	DO_COMPACT(sve_compact_h, uint16_t, H1_2)
3695	DO_COMPACT(sve_compact_s, uint32_t, H1_4)
3696	DO_COMPACT(sve_compact_d, uint64_t, H1_8)
3697
3698	#undef DO_COMPACT
3699
3700	#define DO_EXPAND(NAME, TYPE, H) \
3701	void HELPER(NAME)(void vd, void vn, void *vg, uint32_t desc) \
3702	{ \
3703	intptr_t oprsz = simd_oprsz(desc); \
3704	ARMVectorReg tmp_n = (ARMVectorReg )vn; \
3705	for (intptr_t i = 0, j = 0; i < oprsz; ) { \
3706	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
3707	do { \
3708	TYPE nn = 0; \
3709	if (pg & 1) { \
3710	nn = (TYPE )((void *)&tmp_n + H(j)); \
3711	j += sizeof(TYPE); \
3712	} \
3713	(TYPE )(vd + H(i)) = nn; \
3714	i += sizeof(TYPE); \
3715	pg >>= sizeof(TYPE); \
3716	} while (i & 15); \
3717	} \
3718	}
3719
3720	DO_EXPAND(sve_expand_b, uint8_t, H1)
3721	DO_EXPAND(sve_expand_h, uint16_t, H1_2)
3722	DO_EXPAND(sve_expand_s, uint32_t, H1_4)
3723	DO_EXPAND(sve_expand_d, uint64_t, H1_8)
3724
3725	#undef DO_EXPAND
3726	/* Similar to the ARM LastActiveElement pseudocode function, except the
3727	* result is multiplied by the element size. This includes the not found
3728	* indication; e.g. not found for esz=3 is -8.
3729	*/
3730	int32_t HELPER(sve_last_active_element)(void *vg, uint32_t pred_desc)
3731	{
3732	intptr_t words = DIV_ROUND_UP(FIELD_EX32(pred_desc, PREDDESC, OPRSZ), 8);
3733	intptr_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
3734
3735	return last_active_element(vg, words, esz);
3736	}
3737
3738	void HELPER(sve_splice)(void vd, void vn, void vm, void vg, uint32_t desc)
3739	{
3740	intptr_t opr_sz = simd_oprsz(desc) / 8;
3741	int esz = simd_data(desc);
3742	uint64_t pg, first_g, last_g, len, mask = pred_esz_masks[esz];
3743	intptr_t i, first_i, last_i;
3744	ARMVectorReg tmp;
3745
3746	first_i = last_i = 0;
3747	first_g = last_g = 0;
3748
3749	/* Find the extent of the active elements within VG. */
3750	for (i = QEMU_ALIGN_UP(opr_sz, 8) - 8; i >= 0; i -= 8) {
3751	pg = (uint64_t )(vg + i) & mask;
3752	if (pg) {
3753	if (last_g == 0) {
3754	last_g = pg;
3755	last_i = i;
3756	}
3757	first_g = pg;
3758	first_i = i;
3759	}
3760	}
3761
3762	len = 0;
3763	if (first_g != 0) {
3764	first_i = first_i * 8 + ctz64(first_g);
3765	last_i = last_i * 8 + 63 - clz64(last_g);
3766	len = last_i - first_i + (1 << esz);
3767	if (vd == vm) {
3768	vm = memcpy(&tmp, vm, opr_sz * 8);
3769	}
3770	swap_memmove(vd, vn + first_i, len);
3771	}
3772	swap_memmove(vd + len, vm, opr_sz * 8 - len);
3773	}
3774
3775	void HELPER(sve_sel_zpzz_b)(void vd, void vn, void *vm,
3776	void *vg, uint32_t desc)
3777	{
3778	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
3779	uint64_t d = vd, n = vn, *m = vm;
3780	uint8_t *pg = vg;
3781
3782	for (i = 0; i < opr_sz; i += 1) {
3783	uint64_t nn = n[i], mm = m[i];
3784	uint64_t pp = expand_pred_b(pg[H1(i)]);
3785	d[i] = (nn & pp) \| (mm & ~pp);
3786	}
3787	}
3788
3789	void HELPER(sve_sel_zpzz_h)(void vd, void vn, void *vm,
3790	void *vg, uint32_t desc)
3791	{
3792	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
3793	uint64_t d = vd, n = vn, *m = vm;
3794	uint8_t *pg = vg;
3795
3796	for (i = 0; i < opr_sz; i += 1) {
3797	uint64_t nn = n[i], mm = m[i];
3798	uint64_t pp = expand_pred_h(pg[H1(i)]);
3799	d[i] = (nn & pp) \| (mm & ~pp);
3800	}
3801	}
3802
3803	void HELPER(sve_sel_zpzz_s)(void vd, void vn, void *vm,
3804	void *vg, uint32_t desc)
3805	{
3806	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
3807	uint64_t d = vd, n = vn, *m = vm;
3808	uint8_t *pg = vg;
3809
3810	for (i = 0; i < opr_sz; i += 1) {
3811	uint64_t nn = n[i], mm = m[i];
3812	uint64_t pp = expand_pred_s(pg[H1(i)]);
3813	d[i] = (nn & pp) \| (mm & ~pp);
3814	}
3815	}
3816
3817	void HELPER(sve_sel_zpzz_d)(void vd, void vn, void *vm,
3818	void *vg, uint32_t desc)
3819	{
3820	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
3821	uint64_t d = vd, n = vn, *m = vm;
3822	uint8_t *pg = vg;
3823
3824	for (i = 0; i < opr_sz; i += 1) {
3825	uint64_t nn = n[i], mm = m[i];
3826	d[i] = (pg[H1(i)] & 1 ? nn : mm);
3827	}
3828	}
3829
3830	void HELPER(sve_sel_zpzz_q)(void vd, void vn, void *vm,
3831	void *vg, uint32_t desc)
3832	{
3833	intptr_t i, opr_sz = simd_oprsz(desc) / 16;
3834	Int128 d = vd, n = vn, *m = vm;
3835	uint16_t *pg = vg;
3836
3837	for (i = 0; i < opr_sz; i += 1) {
3838	d[i] = (pg[H2(i)] & 1 ? n : m)[i];
3839	}
3840	}
3841
3842	/* Two operand comparison controlled by a predicate.
3843	* ??? It is very tempting to want to be able to expand this inline
3844	* with x86 instructions, e.g.
3845	*
3846	* vcmpeqw zm, zn, %ymm0
3847	* vpmovmskb %ymm0, %eax
3848	* and $0x5555, %eax
3849	* and pg, %eax
3850	*
3851	* or even aarch64, e.g.
3852	*
3853	* // mask = 4000 1000 0400 0100 0040 0010 0004 0001
3854	* cmeq v0.8h, zn, zm
3855	* and v0.8h, v0.8h, mask
3856	* addv h0, v0.8h
3857	* and v0.8b, pg
3858	*
3859	* However, coming up with an abstraction that allows vector inputs and
3860	* a scalar output, and also handles the byte-ordering of sub-uint64_t
3861	* scalar outputs, is tricky.
3862	*/
3863	#define DO_CMP_PPZZ(NAME, TYPE, OP, H, MASK) \
3864	uint32_t HELPER(NAME)(void vd, void vn, void vm, void vg, uint32_t desc) \
3865	{ \
3866	intptr_t opr_sz = simd_oprsz(desc); \
3867	uint32_t flags = PREDTEST_INIT; \
3868	intptr_t i = opr_sz; \
3869	do { \
3870	uint64_t out = 0, pg; \
3871	do { \
3872	i -= sizeof(TYPE), out <<= sizeof(TYPE); \
3873	TYPE nn = (TYPE )(vn + H(i)); \
3874	TYPE mm = (TYPE )(vm + H(i)); \
3875	out \|= nn OP mm; \
3876	} while (i & 63); \
3877	pg = (uint64_t )(vg + (i >> 3)) & MASK; \
3878	out &= pg; \
3879	(uint64_t )(vd + (i >> 3)) = out; \
3880	flags = iter_predtest_bwd(out, pg, flags); \
3881	} while (i > 0); \
3882	return flags; \
3883	}
3884
3885	#define DO_CMP_PPZZ_B(NAME, TYPE, OP) \
3886	DO_CMP_PPZZ(NAME, TYPE, OP, H1, 0xffffffffffffffffull)
3887	#define DO_CMP_PPZZ_H(NAME, TYPE, OP) \
3888	DO_CMP_PPZZ(NAME, TYPE, OP, H1_2, 0x5555555555555555ull)
3889	#define DO_CMP_PPZZ_S(NAME, TYPE, OP) \
3890	DO_CMP_PPZZ(NAME, TYPE, OP, H1_4, 0x1111111111111111ull)
3891	#define DO_CMP_PPZZ_D(NAME, TYPE, OP) \
3892	DO_CMP_PPZZ(NAME, TYPE, OP, H1_8, 0x0101010101010101ull)
3893
3894	DO_CMP_PPZZ_B(sve_cmpeq_ppzz_b, uint8_t, ==)
3895	DO_CMP_PPZZ_H(sve_cmpeq_ppzz_h, uint16_t, ==)
3896	DO_CMP_PPZZ_S(sve_cmpeq_ppzz_s, uint32_t, ==)
3897	DO_CMP_PPZZ_D(sve_cmpeq_ppzz_d, uint64_t, ==)
3898
3899	DO_CMP_PPZZ_B(sve_cmpne_ppzz_b, uint8_t, !=)
3900	DO_CMP_PPZZ_H(sve_cmpne_ppzz_h, uint16_t, !=)
3901	DO_CMP_PPZZ_S(sve_cmpne_ppzz_s, uint32_t, !=)
3902	DO_CMP_PPZZ_D(sve_cmpne_ppzz_d, uint64_t, !=)
3903
3904	DO_CMP_PPZZ_B(sve_cmpgt_ppzz_b, int8_t, >)
3905	DO_CMP_PPZZ_H(sve_cmpgt_ppzz_h, int16_t, >)
3906	DO_CMP_PPZZ_S(sve_cmpgt_ppzz_s, int32_t, >)
3907	DO_CMP_PPZZ_D(sve_cmpgt_ppzz_d, int64_t, >)
3908
3909	DO_CMP_PPZZ_B(sve_cmpge_ppzz_b, int8_t, >=)
3910	DO_CMP_PPZZ_H(sve_cmpge_ppzz_h, int16_t, >=)
3911	DO_CMP_PPZZ_S(sve_cmpge_ppzz_s, int32_t, >=)
3912	DO_CMP_PPZZ_D(sve_cmpge_ppzz_d, int64_t, >=)
3913
3914	DO_CMP_PPZZ_B(sve_cmphi_ppzz_b, uint8_t, >)
3915	DO_CMP_PPZZ_H(sve_cmphi_ppzz_h, uint16_t, >)
3916	DO_CMP_PPZZ_S(sve_cmphi_ppzz_s, uint32_t, >)
3917	DO_CMP_PPZZ_D(sve_cmphi_ppzz_d, uint64_t, >)
3918
3919	DO_CMP_PPZZ_B(sve_cmphs_ppzz_b, uint8_t, >=)
3920	DO_CMP_PPZZ_H(sve_cmphs_ppzz_h, uint16_t, >=)
3921	DO_CMP_PPZZ_S(sve_cmphs_ppzz_s, uint32_t, >=)
3922	DO_CMP_PPZZ_D(sve_cmphs_ppzz_d, uint64_t, >=)
3923
3924	#undef DO_CMP_PPZZ_B
3925	#undef DO_CMP_PPZZ_H
3926	#undef DO_CMP_PPZZ_S
3927	#undef DO_CMP_PPZZ_D
3928	#undef DO_CMP_PPZZ
3929
3930	/* Similar, but the second source is "wide". */
3931	#define DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H, MASK) \
3932	uint32_t HELPER(NAME)(void vd, void vn, void vm, void vg, uint32_t desc) \
3933	{ \
3934	intptr_t opr_sz = simd_oprsz(desc); \
3935	uint32_t flags = PREDTEST_INIT; \
3936	intptr_t i = opr_sz; \
3937	do { \
3938	uint64_t out = 0, pg; \
3939	do { \
3940	TYPEW mm = (TYPEW )(vm + i - 8); \
3941	do { \
3942	i -= sizeof(TYPE), out <<= sizeof(TYPE); \
3943	TYPE nn = (TYPE )(vn + H(i)); \
3944	out \|= nn OP mm; \
3945	} while (i & 7); \
3946	} while (i & 63); \
3947	pg = (uint64_t )(vg + (i >> 3)) & MASK; \
3948	out &= pg; \
3949	(uint64_t )(vd + (i >> 3)) = out; \
3950	flags = iter_predtest_bwd(out, pg, flags); \
3951	} while (i > 0); \
3952	return flags; \
3953	}
3954
3955	#define DO_CMP_PPZW_B(NAME, TYPE, TYPEW, OP) \
3956	DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1, 0xffffffffffffffffull)
3957	#define DO_CMP_PPZW_H(NAME, TYPE, TYPEW, OP) \
3958	DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1_2, 0x5555555555555555ull)
3959	#define DO_CMP_PPZW_S(NAME, TYPE, TYPEW, OP) \
3960	DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1_4, 0x1111111111111111ull)
3961
3962	DO_CMP_PPZW_B(sve_cmpeq_ppzw_b, int8_t, uint64_t, ==)
3963	DO_CMP_PPZW_H(sve_cmpeq_ppzw_h, int16_t, uint64_t, ==)
3964	DO_CMP_PPZW_S(sve_cmpeq_ppzw_s, int32_t, uint64_t, ==)
3965
3966	DO_CMP_PPZW_B(sve_cmpne_ppzw_b, int8_t, uint64_t, !=)
3967	DO_CMP_PPZW_H(sve_cmpne_ppzw_h, int16_t, uint64_t, !=)
3968	DO_CMP_PPZW_S(sve_cmpne_ppzw_s, int32_t, uint64_t, !=)
3969
3970	DO_CMP_PPZW_B(sve_cmpgt_ppzw_b, int8_t, int64_t, >)
3971	DO_CMP_PPZW_H(sve_cmpgt_ppzw_h, int16_t, int64_t, >)
3972	DO_CMP_PPZW_S(sve_cmpgt_ppzw_s, int32_t, int64_t, >)
3973
3974	DO_CMP_PPZW_B(sve_cmpge_ppzw_b, int8_t, int64_t, >=)
3975	DO_CMP_PPZW_H(sve_cmpge_ppzw_h, int16_t, int64_t, >=)
3976	DO_CMP_PPZW_S(sve_cmpge_ppzw_s, int32_t, int64_t, >=)
3977
3978	DO_CMP_PPZW_B(sve_cmphi_ppzw_b, uint8_t, uint64_t, >)
3979	DO_CMP_PPZW_H(sve_cmphi_ppzw_h, uint16_t, uint64_t, >)
3980	DO_CMP_PPZW_S(sve_cmphi_ppzw_s, uint32_t, uint64_t, >)
3981
3982	DO_CMP_PPZW_B(sve_cmphs_ppzw_b, uint8_t, uint64_t, >=)
3983	DO_CMP_PPZW_H(sve_cmphs_ppzw_h, uint16_t, uint64_t, >=)
3984	DO_CMP_PPZW_S(sve_cmphs_ppzw_s, uint32_t, uint64_t, >=)
3985
3986	DO_CMP_PPZW_B(sve_cmplt_ppzw_b, int8_t, int64_t, <)
3987	DO_CMP_PPZW_H(sve_cmplt_ppzw_h, int16_t, int64_t, <)
3988	DO_CMP_PPZW_S(sve_cmplt_ppzw_s, int32_t, int64_t, <)
3989
3990	DO_CMP_PPZW_B(sve_cmple_ppzw_b, int8_t, int64_t, <=)
3991	DO_CMP_PPZW_H(sve_cmple_ppzw_h, int16_t, int64_t, <=)
3992	DO_CMP_PPZW_S(sve_cmple_ppzw_s, int32_t, int64_t, <=)
3993
3994	DO_CMP_PPZW_B(sve_cmplo_ppzw_b, uint8_t, uint64_t, <)
3995	DO_CMP_PPZW_H(sve_cmplo_ppzw_h, uint16_t, uint64_t, <)
3996	DO_CMP_PPZW_S(sve_cmplo_ppzw_s, uint32_t, uint64_t, <)
3997
3998	DO_CMP_PPZW_B(sve_cmpls_ppzw_b, uint8_t, uint64_t, <=)
3999	DO_CMP_PPZW_H(sve_cmpls_ppzw_h, uint16_t, uint64_t, <=)
4000	DO_CMP_PPZW_S(sve_cmpls_ppzw_s, uint32_t, uint64_t, <=)
4001
4002	#undef DO_CMP_PPZW_B
4003	#undef DO_CMP_PPZW_H
4004	#undef DO_CMP_PPZW_S
4005	#undef DO_CMP_PPZW
4006
4007	/* Similar, but the second source is immediate. */
4008	#define DO_CMP_PPZI(NAME, TYPE, OP, H, MASK) \
4009	uint32_t HELPER(NAME)(void vd, void vn, void *vg, uint32_t desc) \
4010	{ \
4011	intptr_t opr_sz = simd_oprsz(desc); \
4012	uint32_t flags = PREDTEST_INIT; \
4013	TYPE mm = simd_data(desc); \
4014	intptr_t i = opr_sz; \
4015	do { \
4016	uint64_t out = 0, pg; \
4017	do { \
4018	i -= sizeof(TYPE), out <<= sizeof(TYPE); \
4019	TYPE nn = (TYPE )(vn + H(i)); \
4020	out \|= nn OP mm; \
4021	} while (i & 63); \
4022	pg = (uint64_t )(vg + (i >> 3)) & MASK; \
4023	out &= pg; \
4024	(uint64_t )(vd + (i >> 3)) = out; \
4025	flags = iter_predtest_bwd(out, pg, flags); \
4026	} while (i > 0); \
4027	return flags; \
4028	}
4029
4030	#define DO_CMP_PPZI_B(NAME, TYPE, OP) \
4031	DO_CMP_PPZI(NAME, TYPE, OP, H1, 0xffffffffffffffffull)
4032	#define DO_CMP_PPZI_H(NAME, TYPE, OP) \
4033	DO_CMP_PPZI(NAME, TYPE, OP, H1_2, 0x5555555555555555ull)
4034	#define DO_CMP_PPZI_S(NAME, TYPE, OP) \
4035	DO_CMP_PPZI(NAME, TYPE, OP, H1_4, 0x1111111111111111ull)
4036	#define DO_CMP_PPZI_D(NAME, TYPE, OP) \
4037	DO_CMP_PPZI(NAME, TYPE, OP, H1_8, 0x0101010101010101ull)
4038
4039	DO_CMP_PPZI_B(sve_cmpeq_ppzi_b, uint8_t, ==)
4040	DO_CMP_PPZI_H(sve_cmpeq_ppzi_h, uint16_t, ==)
4041	DO_CMP_PPZI_S(sve_cmpeq_ppzi_s, uint32_t, ==)
4042	DO_CMP_PPZI_D(sve_cmpeq_ppzi_d, uint64_t, ==)
4043
4044	DO_CMP_PPZI_B(sve_cmpne_ppzi_b, uint8_t, !=)
4045	DO_CMP_PPZI_H(sve_cmpne_ppzi_h, uint16_t, !=)
4046	DO_CMP_PPZI_S(sve_cmpne_ppzi_s, uint32_t, !=)
4047	DO_CMP_PPZI_D(sve_cmpne_ppzi_d, uint64_t, !=)
4048
4049	DO_CMP_PPZI_B(sve_cmpgt_ppzi_b, int8_t, >)
4050	DO_CMP_PPZI_H(sve_cmpgt_ppzi_h, int16_t, >)
4051	DO_CMP_PPZI_S(sve_cmpgt_ppzi_s, int32_t, >)
4052	DO_CMP_PPZI_D(sve_cmpgt_ppzi_d, int64_t, >)
4053
4054	DO_CMP_PPZI_B(sve_cmpge_ppzi_b, int8_t, >=)
4055	DO_CMP_PPZI_H(sve_cmpge_ppzi_h, int16_t, >=)
4056	DO_CMP_PPZI_S(sve_cmpge_ppzi_s, int32_t, >=)
4057	DO_CMP_PPZI_D(sve_cmpge_ppzi_d, int64_t, >=)
4058
4059	DO_CMP_PPZI_B(sve_cmphi_ppzi_b, uint8_t, >)
4060	DO_CMP_PPZI_H(sve_cmphi_ppzi_h, uint16_t, >)
4061	DO_CMP_PPZI_S(sve_cmphi_ppzi_s, uint32_t, >)
4062	DO_CMP_PPZI_D(sve_cmphi_ppzi_d, uint64_t, >)
4063
4064	DO_CMP_PPZI_B(sve_cmphs_ppzi_b, uint8_t, >=)
4065	DO_CMP_PPZI_H(sve_cmphs_ppzi_h, uint16_t, >=)
4066	DO_CMP_PPZI_S(sve_cmphs_ppzi_s, uint32_t, >=)
4067	DO_CMP_PPZI_D(sve_cmphs_ppzi_d, uint64_t, >=)
4068
4069	DO_CMP_PPZI_B(sve_cmplt_ppzi_b, int8_t, <)
4070	DO_CMP_PPZI_H(sve_cmplt_ppzi_h, int16_t, <)
4071	DO_CMP_PPZI_S(sve_cmplt_ppzi_s, int32_t, <)
4072	DO_CMP_PPZI_D(sve_cmplt_ppzi_d, int64_t, <)
4073
4074	DO_CMP_PPZI_B(sve_cmple_ppzi_b, int8_t, <=)
4075	DO_CMP_PPZI_H(sve_cmple_ppzi_h, int16_t, <=)
4076	DO_CMP_PPZI_S(sve_cmple_ppzi_s, int32_t, <=)
4077	DO_CMP_PPZI_D(sve_cmple_ppzi_d, int64_t, <=)
4078
4079	DO_CMP_PPZI_B(sve_cmplo_ppzi_b, uint8_t, <)
4080	DO_CMP_PPZI_H(sve_cmplo_ppzi_h, uint16_t, <)
4081	DO_CMP_PPZI_S(sve_cmplo_ppzi_s, uint32_t, <)
4082	DO_CMP_PPZI_D(sve_cmplo_ppzi_d, uint64_t, <)
4083
4084	DO_CMP_PPZI_B(sve_cmpls_ppzi_b, uint8_t, <=)
4085	DO_CMP_PPZI_H(sve_cmpls_ppzi_h, uint16_t, <=)
4086	DO_CMP_PPZI_S(sve_cmpls_ppzi_s, uint32_t, <=)
4087	DO_CMP_PPZI_D(sve_cmpls_ppzi_d, uint64_t, <=)
4088
4089	#undef DO_CMP_PPZI_B
4090	#undef DO_CMP_PPZI_H
4091	#undef DO_CMP_PPZI_S
4092	#undef DO_CMP_PPZI_D
4093	#undef DO_CMP_PPZI
4094
4095	/* Similar to the ARM LastActive pseudocode function. */
4096	static bool last_active_pred(void vd, void vg, intptr_t oprsz)
4097	{
4098	intptr_t i;
4099
4100	for (i = QEMU_ALIGN_UP(oprsz, 8) - 8; i >= 0; i -= 8) {
4101	uint64_t pg = (uint64_t )(vg + i);
4102	if (pg) {
4103	return (pow2floor(pg) & (uint64_t )(vd + i)) != 0;
4104	}
4105	}
4106	return 0;
4107	}
4108
4109	/* Compute a mask into RETB that is true for all G, up to and including
4110	* (if after) or excluding (if !after) the first G & N.
4111	* Return true if BRK found.
4112	*/
4113	static bool compute_brk(uint64_t *retb, uint64_t n, uint64_t g,
4114	bool brk, bool after)
4115	{
4116	uint64_t b;
4117
4118	if (brk) {
4119	b = 0;
4120	} else if ((g & n) == 0) {
4121	/* For all G, no N are set; break not found. */
4122	b = g;
4123	} else {
4124	/* Break somewhere in N. Locate it. */
4125	b = g & n; /* guard true, pred true */
4126	b = b & -b; /* first such */
4127	if (after) {
4128	b = b \| (b - 1); /* break after same */
4129	} else {
4130	b = b - 1; /* break before same */
4131	}
4132	brk = true;
4133	}
4134
4135	*retb = b;
4136	return brk;
4137	}
4138
4139	/* Compute a zeroing BRK. */
4140	static void compute_brk_z(uint64_t d, uint64_t n, uint64_t *g,
4141	intptr_t oprsz, bool after)
4142	{
4143	bool brk = false;
4144	intptr_t i;
4145
4146	for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
4147	uint64_t this_b, this_g = g[i];
4148
4149	brk = compute_brk(&this_b, n[i], this_g, brk, after);
4150	d[i] = this_b & this_g;
4151	}
4152	}
4153
4154	/* Likewise, but also compute flags. */
4155	static uint32_t compute_brks_z(uint64_t d, uint64_t n, uint64_t *g,
4156	intptr_t oprsz, bool after)
4157	{
4158	uint32_t flags = PREDTEST_INIT;
4159	bool brk = false;
4160	intptr_t i;
4161
4162	for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
4163	uint64_t this_b, this_d, this_g = g[i];
4164
4165	brk = compute_brk(&this_b, n[i], this_g, brk, after);
4166	d[i] = this_d = this_b & this_g;
4167	flags = iter_predtest_fwd(this_d, this_g, flags);
4168	}
4169	return flags;
4170	}
4171
4172	/* Compute a merging BRK. */
4173	static void compute_brk_m(uint64_t d, uint64_t n, uint64_t *g,
4174	intptr_t oprsz, bool after)
4175	{
4176	bool brk = false;
4177	intptr_t i;
4178
4179	for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
4180	uint64_t this_b, this_g = g[i];
4181
4182	brk = compute_brk(&this_b, n[i], this_g, brk, after);
4183	d[i] = (this_b & this_g) \| (d[i] & ~this_g);
4184	}
4185	}
4186
4187	/* Likewise, but also compute flags. */
4188	static uint32_t compute_brks_m(uint64_t d, uint64_t n, uint64_t *g,
4189	intptr_t oprsz, bool after)
4190	{
4191	uint32_t flags = PREDTEST_INIT;
4192	bool brk = false;
4193	intptr_t i;
4194
4195	for (i = 0; i < oprsz / 8; ++i) {
4196	uint64_t this_b, this_d = d[i], this_g = g[i];
4197
4198	brk = compute_brk(&this_b, n[i], this_g, brk, after);
4199	d[i] = this_d = (this_b & this_g) \| (this_d & ~this_g);
4200	flags = iter_predtest_fwd(this_d, this_g, flags);
4201	}
4202	return flags;
4203	}
4204
4205	void HELPER(sve_brkpa)(void vd, void vn, void vm, void vg,
4206	uint32_t pred_desc)
4207	{
4208	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4209	if (last_active_pred(vn, vg, oprsz)) {
4210	compute_brk_z(vd, vm, vg, oprsz, true);
4211	} else {
4212	memset(vd, 0, sizeof(ARMPredicateReg));
4213	}
4214	}
4215
4216	uint32_t HELPER(sve_brkpas)(void vd, void vn, void vm, void vg,
4217	uint32_t pred_desc)
4218	{
4219	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4220	if (last_active_pred(vn, vg, oprsz)) {
4221	return compute_brks_z(vd, vm, vg, oprsz, true);
4222	} else {
4223	memset(vd, 0, sizeof(ARMPredicateReg));
4224	return PREDTEST_INIT;
4225	}
4226	}
4227
4228	void HELPER(sve_brkpb)(void vd, void vn, void vm, void vg,
4229	uint32_t pred_desc)
4230	{
4231	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4232	if (last_active_pred(vn, vg, oprsz)) {
4233	compute_brk_z(vd, vm, vg, oprsz, false);
4234	} else {
4235	memset(vd, 0, sizeof(ARMPredicateReg));
4236	}
4237	}
4238
4239	uint32_t HELPER(sve_brkpbs)(void vd, void vn, void vm, void vg,
4240	uint32_t pred_desc)
4241	{
4242	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4243	if (last_active_pred(vn, vg, oprsz)) {
4244	return compute_brks_z(vd, vm, vg, oprsz, false);
4245	} else {
4246	memset(vd, 0, sizeof(ARMPredicateReg));
4247	return PREDTEST_INIT;
4248	}
4249	}
4250
4251	void HELPER(sve_brka_z)(void vd, void vn, void *vg, uint32_t pred_desc)
4252	{
4253	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4254	compute_brk_z(vd, vn, vg, oprsz, true);
4255	}
4256
4257	uint32_t HELPER(sve_brkas_z)(void vd, void vn, void *vg, uint32_t pred_desc)
4258	{
4259	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4260	return compute_brks_z(vd, vn, vg, oprsz, true);
4261	}
4262
4263	void HELPER(sve_brkb_z)(void vd, void vn, void *vg, uint32_t pred_desc)
4264	{
4265	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4266	compute_brk_z(vd, vn, vg, oprsz, false);
4267	}
4268
4269	uint32_t HELPER(sve_brkbs_z)(void vd, void vn, void *vg, uint32_t pred_desc)
4270	{
4271	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4272	return compute_brks_z(vd, vn, vg, oprsz, false);
4273	}
4274
4275	void HELPER(sve_brka_m)(void vd, void vn, void *vg, uint32_t pred_desc)
4276	{
4277	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4278	compute_brk_m(vd, vn, vg, oprsz, true);
4279	}
4280
4281	uint32_t HELPER(sve_brkas_m)(void vd, void vn, void *vg, uint32_t pred_desc)
4282	{
4283	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4284	return compute_brks_m(vd, vn, vg, oprsz, true);
4285	}
4286
4287	void HELPER(sve_brkb_m)(void vd, void vn, void *vg, uint32_t pred_desc)
4288	{
4289	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4290	compute_brk_m(vd, vn, vg, oprsz, false);
4291	}
4292
4293	uint32_t HELPER(sve_brkbs_m)(void vd, void vn, void *vg, uint32_t pred_desc)
4294	{
4295	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4296	return compute_brks_m(vd, vn, vg, oprsz, false);
4297	}
4298
4299	void HELPER(sve_brkn)(void vd, void vn, void *vg, uint32_t pred_desc)
4300	{
4301	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4302	if (!last_active_pred(vn, vg, oprsz)) {
4303	memset(vd, 0, sizeof(ARMPredicateReg));
4304	}
4305	}
4306
4307	uint32_t HELPER(sve_brkns)(void vd, void vn, void *vg, uint32_t pred_desc)
4308	{
4309	intptr_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4310	if (last_active_pred(vn, vg, oprsz)) {
4311	ARMPredicateReg *d = vd;
4312	uint32_t flags = PREDTEST_INIT;
4313	intptr_t i;
4314
4315	/* As if PredTest(Ones(PL), D, MO_8). */
4316	for (i = 0; i < oprsz / 8; i++) {
4317	flags = iter_predtest_fwd(d->p[i], -1, flags);
4318	}
4319	if (oprsz & 7) {
4320	uint64_t mask = ~(-1ULL << (8 * (oprsz & 7)));
4321	flags = iter_predtest_fwd(d->p[i], mask, flags);
4322	}
4323	return flags;
4324	}
4325	memset(vd, 0, sizeof(ARMPredicateReg));
4326	return PREDTEST_INIT;
4327	}
4328
4329	uint64_t HELPER(sve_cntp)(void vn, void vg, uint32_t pred_desc)
4330	{
4331	intptr_t words = DIV_ROUND_UP(FIELD_EX32(pred_desc, PREDDESC, OPRSZ), 8);
4332	intptr_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4333	uint64_t n = vn, g = vg, sum = 0, mask = pred_esz_masks[esz];
4334	intptr_t i;
4335
4336	for (i = 0; i < words; ++i) {
4337	uint64_t t = n[i] & g[i] & mask;
4338	sum += ctpop64(t);
4339	}
4340	return sum;
4341	}
4342
4343	uint64_t HELPER(sve2p1_cntp_c)(uint32_t png, uint32_t desc)
4344	{
4345	int pl = FIELD_EX32(desc, PREDDESC, OPRSZ);
4346	int vl = pl * 8;
4347	unsigned v_esz = FIELD_EX32(desc, PREDDESC, ESZ);
4348	int lg2_width = FIELD_EX32(desc, PREDDESC, DATA) + 1;
4349	DecodeCounter p = decode_counter(png, vl, v_esz);
4350	unsigned maxelem = (vl << lg2_width) >> v_esz;
4351	unsigned count = p.count;
4352
4353	if (p.invert) {
4354	if (count >= maxelem) {
4355	return 0;
4356	}
4357	count = maxelem - count;
4358	} else {
4359	count = MIN(count, maxelem);
4360	}
4361	return count >> p.lg2_stride;
4362	}
4363
4364	uint64_t HELPER(sve_firstp)(void vn, void vg, uint32_t pred_desc)
4365	{
4366	intptr_t words = DIV_ROUND_UP(FIELD_EX32(pred_desc, PREDDESC, OPRSZ), 8);
4367	intptr_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4368	uint64_t n = vn, g = vg, mask = pred_esz_masks[esz];
4369
4370	for (intptr_t i = 0; i < words; ++i) {
4371	uint64_t t = n[i] & g[i] & mask;
4372	if (t) {
4373	return (i * 64 + ctz64(t)) >> esz;
4374	}
4375	}
4376	return -1;
4377	}
4378
4379	uint64_t HELPER(sve_lastp)(void vn, void vg, uint32_t pred_desc)
4380	{
4381	intptr_t words = DIV_ROUND_UP(FIELD_EX32(pred_desc, PREDDESC, OPRSZ), 8);
4382	intptr_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4383	uint64_t n = vn, g = vg, mask = pred_esz_masks[esz];
4384
4385	for (intptr_t i = words - 1; i >= 0; --i) {
4386	uint64_t t = n[i] & g[i] & mask;
4387	if (t) {
4388	return (i * 64 + (63 - clz64(t))) >> esz;
4389	}
4390	}
4391	return -1;
4392	}
4393
4394	/* C.f. Arm pseudocode EncodePredCount */
4395	static uint64_t encode_pred_count(uint32_t elements, uint32_t count,
4396	uint32_t esz, bool invert)
4397	{
4398	uint32_t pred;
4399
4400	if (count == 0) {
4401	return 0;
4402	}
4403	if (invert) {
4404	count = elements - count;
4405	} else if (count == elements) {
4406	count = 0;
4407	invert = true;
4408	}
4409
4410	pred = (count << 1) \| 1;
4411	pred <<= esz;
4412	pred \|= invert << 15;
4413
4414	return pred;
4415	}
4416
4417	/* C.f. Arm pseudocode PredCountTest */
4418	static uint32_t pred_count_test(uint32_t elements, uint32_t count, bool invert)
4419	{
4420	uint32_t flags;
4421
4422	if (count == 0) {
4423	flags = 1; /* !N, Z, C */
4424	} else if (!invert) {
4425	flags = (1u << 31) \| 2; /* N, !Z */
4426	flags \|= count != elements; /* C */
4427	} else {
4428	flags = 2; /* !Z, !C */
4429	flags \|= (count == elements) << 31; /* N */
4430	}
4431	return flags;
4432	}
4433
4434	/* D must be cleared on entry. */
4435	static void do_whilel(ARMPredicateReg *d, uint64_t esz_mask,
4436	uint32_t count, uint32_t oprbits)
4437	{
4438	tcg_debug_assert(count <= oprbits);
4439	if (count) {
4440	uint32_t i;
4441
4442	/* Set all of the requested bits. */
4443	for (i = 0; i < count / 64; ++i) {
4444	d->p[i] = esz_mask;
4445	}
4446	if (count & 63) {
4447	d->p[i] = MAKE_64BIT_MASK(0, count & 63) & esz_mask;
4448	}
4449	}
4450	}
4451
4452	uint32_t HELPER(sve_whilel)(void *vd, uint32_t count, uint32_t pred_desc)
4453	{
4454	uint32_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4455	uint32_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4456	uint32_t oprbits = oprsz * 8;
4457	uint64_t esz_mask = pred_esz_masks[esz];
4458	ARMPredicateReg *d = vd;
4459
4460	count <<= esz;
4461	memset(d, 0, sizeof(*d));
4462	do_whilel(d, esz_mask, count, oprbits);
4463	return pred_count_test(oprbits, count, false);
4464	}
4465
4466	uint32_t HELPER(sve_while2l)(void *vd, uint32_t count, uint32_t pred_desc)
4467	{
4468	uint32_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4469	uint32_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4470	uint32_t oprbits = oprsz * 8;
4471	uint64_t esz_mask = pred_esz_masks[esz];
4472	ARMPredicateReg *d = vd;
4473
4474	count <<= esz;
4475	memset(d, 0, 2 * sizeof(*d));
4476	if (count <= oprbits) {
4477	do_whilel(&d[0], esz_mask, count, oprbits);
4478	} else {
4479	do_whilel(&d[0], esz_mask, oprbits, oprbits);
4480	do_whilel(&d[1], esz_mask, count - oprbits, oprbits);
4481	}
4482
4483	return pred_count_test(2 * oprbits, count, false);
4484	}
4485
4486	uint32_t HELPER(sve_whilecl)(void *vd, uint32_t count, uint32_t pred_desc)
4487	{
4488	uint32_t pl = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4489	uint32_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4490	uint32_t scale = FIELD_EX32(pred_desc, PREDDESC, DATA);
4491	uint32_t vl = pl * 8;
4492	uint32_t elements = (vl >> esz) << scale;
4493	ARMPredicateReg *d = vd;
4494
4495	*d = (ARMPredicateReg) {
4496	.p[0] = encode_pred_count(elements, count, esz, false)
4497	};
4498	return pred_count_test(elements, count, false);
4499	}
4500
4501	/* D must be cleared on entry. */
4502	static void do_whileg(ARMPredicateReg *d, uint64_t esz_mask,
4503	uint32_t count, uint32_t oprbits)
4504	{
4505	tcg_debug_assert(count <= oprbits);
4506	if (count) {
4507	uint32_t i, invcount = oprbits - count;
4508	uint64_t bits = esz_mask & MAKE_64BIT_MASK(invcount & 63, 64);
4509
4510	for (i = invcount / 64; i < oprbits / 64; ++i) {
4511	d->p[i] = bits;
4512	bits = esz_mask;
4513	}
4514	if (oprbits & 63) {
4515	d->p[i] = bits & MAKE_64BIT_MASK(0, oprbits & 63);
4516	}
4517	}
4518	}
4519
4520	uint32_t HELPER(sve_whileg)(void *vd, uint32_t count, uint32_t pred_desc)
4521	{
4522	uint32_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4523	uint32_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4524	uint32_t oprbits = oprsz * 8;
4525	uint64_t esz_mask = pred_esz_masks[esz];
4526	ARMPredicateReg *d = vd;
4527
4528	count <<= esz;
4529	memset(d, 0, sizeof(*d));
4530	do_whileg(d, esz_mask, count, oprbits);
4531	return pred_count_test(oprbits, count, true);
4532	}
4533
4534	uint32_t HELPER(sve_while2g)(void *vd, uint32_t count, uint32_t pred_desc)
4535	{
4536	uint32_t oprsz = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4537	uint32_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4538	uint32_t oprbits = oprsz * 8;
4539	uint64_t esz_mask = pred_esz_masks[esz];
4540	ARMPredicateReg *d = vd;
4541
4542	count <<= esz;
4543	memset(d, 0, 2 * sizeof(*d));
4544	if (count <= oprbits) {
4545	do_whileg(&d[1], esz_mask, count, oprbits);
4546	} else {
4547	do_whilel(&d[1], esz_mask, oprbits, oprbits);
4548	do_whileg(&d[0], esz_mask, count - oprbits, oprbits);
4549	}
4550
4551	return pred_count_test(2 * oprbits, count, true);
4552	}
4553
4554	uint32_t HELPER(sve_whilecg)(void *vd, uint32_t count, uint32_t pred_desc)
4555	{
4556	uint32_t pl = FIELD_EX32(pred_desc, PREDDESC, OPRSZ);
4557	uint32_t esz = FIELD_EX32(pred_desc, PREDDESC, ESZ);
4558	uint32_t scale = FIELD_EX32(pred_desc, PREDDESC, DATA);
4559	uint32_t vl = pl * 8;
4560	uint32_t elements = (vl >> esz) << scale;
4561	ARMPredicateReg *d = vd;
4562
4563	*d = (ARMPredicateReg) {
4564	.p[0] = encode_pred_count(elements, count, esz, true)
4565	};
4566	return pred_count_test(elements, count, true);
4567	}
4568
4569	/* Recursive reduction on a function;
4570	* C.f. the ARM ARM function ReducePredicated.
4571	*
4572	* While it would be possible to write this without the DATA temporary,
4573	* it is much simpler to process the predicate register this way.
4574	* The recursion is bounded to depth 7 (128 fp16 elements), so there's
4575	* little to gain with a more complex non-recursive form.
4576	*/
4577	#define DO_REDUCE(NAME, SUF, TYPE, H, FUNC, IDENT) \
4578	static TYPE FUNC##_reduce(TYPE data, float_status status, uintptr_t n) \
4579	{ \
4580	if (n == 1) { \
4581	return *data; \
4582	} else { \
4583	uintptr_t half = n / 2; \
4584	TYPE lo = FUNC##_reduce(data, status, half); \
4585	TYPE hi = FUNC##_reduce(data + half, status, half); \
4586	return FUNC(lo, hi, status); \
4587	} \
4588	} \
4589	uint64_t helper_sve_##NAME##v_##SUF(void vn, void vg, \
4590	float_status *status, uint32_t desc) \
4591	{ \
4592	uintptr_t i, oprsz = simd_oprsz(desc), maxsz = simd_data(desc); \
4593	TYPE data[sizeof(ARMVectorReg) / sizeof(TYPE)]; \
4594	TYPE ident = IDENT; \
4595	for (i = 0; i < oprsz; ) { \
4596	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3)); \
4597	do { \
4598	TYPE nn = (TYPE )(vn + H(i)); \
4599	(TYPE )((void *)data + i) = (pg & 1 ? nn : ident); \
4600	i += sizeof(TYPE), pg >>= sizeof(TYPE); \
4601	} while (i & 15); \
4602	} \
4603	for (; i < maxsz; i += sizeof(TYPE)) { \
4604	(TYPE )((void *)data + i) = ident; \
4605	} \
4606	return FUNC##_reduce(data, status, maxsz / sizeof(TYPE)); \
4607	} \
4608	void helper_sve2p1_##NAME##qv_##SUF(void vd, void vn, void *vg, \
4609	float_status *status, uint32_t desc) \
4610	{ \
4611	unsigned oprsz = simd_oprsz(desc), segments = oprsz / 16; \
4612	TYPE ident = IDENT; \
4613	for (unsigned e = 0; e < 16; e += sizeof(TYPE)) { \
4614	TYPE data[ARM_MAX_VQ]; \
4615	for (unsigned s = 0; s < segments; s++) { \
4616	uint16_t pg = (uint16_t )(vg + H1_2(s * 2)); \
4617	TYPE nn = (TYPE )(vn + (s * 16 + H(e))); \
4618	data[s] = (pg >> e) & 1 ? nn : ident; \
4619	} \
4620	(TYPE )(vd + H(e)) = FUNC##_reduce(data, status, segments); \
4621	} \
4622	clear_tail(vd, 16, simd_maxsz(desc)); \
4623	}
4624
4625	DO_REDUCE(fadd,h, float16, H1_2, float16_add, float16_zero)
4626	DO_REDUCE(fadd,s, float32, H1_4, float32_add, float32_zero)
4627	DO_REDUCE(fadd,d, float64, H1_8, float64_add, float64_zero)
4628
4629	/*
4630	* We can't avoid the function call for the default NaN value, because
4631	* it changes when FPCR.AH is set.
4632	*/
4633	DO_REDUCE(fminnm,h, float16, H1_2, float16_minnum, float16_default_nan(status))
4634	DO_REDUCE(fminnm,s, float32, H1_4, float32_minnum, float32_default_nan(status))
4635	DO_REDUCE(fminnm,d, float64, H1_8, float64_minnum, float64_default_nan(status))
4636
4637	DO_REDUCE(fmaxnm,h, float16, H1_2, float16_maxnum, float16_default_nan(status))
4638	DO_REDUCE(fmaxnm,s, float32, H1_4, float32_maxnum, float32_default_nan(status))
4639	DO_REDUCE(fmaxnm,d, float64, H1_8, float64_maxnum, float64_default_nan(status))
4640
4641	DO_REDUCE(fmin,h, float16, H1_2, float16_min, float16_infinity)
4642	DO_REDUCE(fmin,s, float32, H1_4, float32_min, float32_infinity)
4643	DO_REDUCE(fmin,d, float64, H1_8, float64_min, float64_infinity)
4644
4645	DO_REDUCE(fmax,h, float16, H1_2, float16_max, float16_chs(float16_infinity))
4646	DO_REDUCE(fmax,s, float32, H1_4, float32_max, float32_chs(float32_infinity))
4647	DO_REDUCE(fmax,d, float64, H1_8, float64_max, float64_chs(float64_infinity))
4648
4649	DO_REDUCE(ah_fmin,h, float16, H1_2, helper_vfp_ah_minh, float16_infinity)
4650	DO_REDUCE(ah_fmin,s, float32, H1_4, helper_vfp_ah_mins, float32_infinity)
4651	DO_REDUCE(ah_fmin,d, float64, H1_8, helper_vfp_ah_mind, float64_infinity)
4652
4653	DO_REDUCE(ah_fmax,h, float16, H1_2, helper_vfp_ah_maxh,
4654	float16_chs(float16_infinity))
4655	DO_REDUCE(ah_fmax,s, float32, H1_4, helper_vfp_ah_maxs,
4656	float32_chs(float32_infinity))
4657	DO_REDUCE(ah_fmax,d, float64, H1_8, helper_vfp_ah_maxd,
4658	float64_chs(float64_infinity))
4659
4660	#undef DO_REDUCE
4661
4662	uint64_t HELPER(sve_fadda_h)(uint64_t nn, void vm, void vg,
4663	float_status *status, uint32_t desc)
4664	{
4665	intptr_t i = 0, opr_sz = simd_oprsz(desc);
4666	float16 result = nn;
4667
4668	do {
4669	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3));
4670	do {
4671	if (pg & 1) {
4672	float16 mm = (float16 )(vm + H1_2(i));
4673	result = float16_add(result, mm, status);
4674	}
4675	i += sizeof(float16), pg >>= sizeof(float16);
4676	} while (i & 15);
4677	} while (i < opr_sz);
4678
4679	return result;
4680	}
4681
4682	uint64_t HELPER(sve_fadda_s)(uint64_t nn, void vm, void vg,
4683	float_status *status, uint32_t desc)
4684	{
4685	intptr_t i = 0, opr_sz = simd_oprsz(desc);
4686	float32 result = nn;
4687
4688	do {
4689	uint16_t pg = (uint16_t )(vg + H1_2(i >> 3));
4690	do {
4691	if (pg & 1) {
4692	float32 mm = (float32 )(vm + H1_2(i));
4693	result = float32_add(result, mm, status);
4694	}
4695	i += sizeof(float32), pg >>= sizeof(float32);
4696	} while (i & 15);
4697	} while (i < opr_sz);
4698
4699	return result;
4700	}
4701
4702	uint64_t HELPER(sve_fadda_d)(uint64_t nn, void vm, void vg,
4703	float_status *status, uint32_t desc)
4704	{
4705	intptr_t i = 0, opr_sz = simd_oprsz(desc) / 8;
4706	uint64_t *m = vm;
4707	uint8_t *pg = vg;
4708
4709	for (i = 0; i < opr_sz; i++) {
4710	if (pg[H1(i)] & 1) {
4711	nn = float64_add(nn, m[i], status);
4712	}
4713	}
4714
4715	return nn;
4716	}
4717
4718	/* Fully general three-operand expander, controlled by a predicate,
4719	* With the extra float_status parameter.
4720	*/
4721	#define DO_ZPZZ_FP(NAME, TYPE, H, OP) \
4722	void HELPER(NAME)(void vd, void vn, void vm, void vg, \
4723	float_status *status, uint32_t desc) \
4724	{ \
4725	intptr_t i = simd_oprsz(desc); \
4726	uint64_t *g = vg; \
4727	do { \
4728	uint64_t pg = g[(i - 1) >> 6]; \
4729	do { \
4730	i -= sizeof(TYPE); \
4731	if (likely((pg >> (i & 63)) & 1)) { \
4732	TYPE nn = (TYPE )(vn + H(i)); \
4733	TYPE mm = (TYPE )(vm + H(i)); \
4734	(TYPE )(vd + H(i)) = OP(nn, mm, status); \
4735	} \
4736	} while (i & 63); \
4737	} while (i != 0); \
4738	}
4739
4740	DO_ZPZZ_FP(sve_fadd_b16, uint16_t, H1_2, bfloat16_add)
4741	DO_ZPZZ_FP(sve_fadd_h, uint16_t, H1_2, float16_add)
4742	DO_ZPZZ_FP(sve_fadd_s, uint32_t, H1_4, float32_add)
4743	DO_ZPZZ_FP(sve_fadd_d, uint64_t, H1_8, float64_add)
4744
4745	DO_ZPZZ_FP(sve_fsub_b16, uint16_t, H1_2, bfloat16_sub)
4746	DO_ZPZZ_FP(sve_fsub_h, uint16_t, H1_2, float16_sub)
4747	DO_ZPZZ_FP(sve_fsub_s, uint32_t, H1_4, float32_sub)
4748	DO_ZPZZ_FP(sve_fsub_d, uint64_t, H1_8, float64_sub)
4749
4750	DO_ZPZZ_FP(sve_fmul_b16, uint16_t, H1_2, bfloat16_mul)
4751	DO_ZPZZ_FP(sve_fmul_h, uint16_t, H1_2, float16_mul)
4752	DO_ZPZZ_FP(sve_fmul_s, uint32_t, H1_4, float32_mul)
4753	DO_ZPZZ_FP(sve_fmul_d, uint64_t, H1_8, float64_mul)
4754
4755	DO_ZPZZ_FP(sve_fdiv_h, uint16_t, H1_2, float16_div)
4756	DO_ZPZZ_FP(sve_fdiv_s, uint32_t, H1_4, float32_div)
4757	DO_ZPZZ_FP(sve_fdiv_d, uint64_t, H1_8, float64_div)
4758
4759	DO_ZPZZ_FP(sve_fmin_b16, uint16_t, H1_2, bfloat16_min)
4760	DO_ZPZZ_FP(sve_fmin_h, uint16_t, H1_2, float16_min)
4761	DO_ZPZZ_FP(sve_fmin_s, uint32_t, H1_4, float32_min)
4762	DO_ZPZZ_FP(sve_fmin_d, uint64_t, H1_8, float64_min)
4763
4764	DO_ZPZZ_FP(sve_fmax_b16, uint16_t, H1_2, bfloat16_max)
4765	DO_ZPZZ_FP(sve_fmax_h, uint16_t, H1_2, float16_max)
4766	DO_ZPZZ_FP(sve_fmax_s, uint32_t, H1_4, float32_max)
4767	DO_ZPZZ_FP(sve_fmax_d, uint64_t, H1_8, float64_max)
4768
4769	DO_ZPZZ_FP(sve_ah_fmin_b16, uint16_t, H1_2, helper_sme2_ah_fmin_b16)
4770	DO_ZPZZ_FP(sve_ah_fmin_h, uint16_t, H1_2, helper_vfp_ah_minh)
4771	DO_ZPZZ_FP(sve_ah_fmin_s, uint32_t, H1_4, helper_vfp_ah_mins)
4772	DO_ZPZZ_FP(sve_ah_fmin_d, uint64_t, H1_8, helper_vfp_ah_mind)
4773
4774	DO_ZPZZ_FP(sve_ah_fmax_b16, uint16_t, H1_2, helper_sme2_ah_fmax_b16)
4775	DO_ZPZZ_FP(sve_ah_fmax_h, uint16_t, H1_2, helper_vfp_ah_maxh)
4776	DO_ZPZZ_FP(sve_ah_fmax_s, uint32_t, H1_4, helper_vfp_ah_maxs)
4777	DO_ZPZZ_FP(sve_ah_fmax_d, uint64_t, H1_8, helper_vfp_ah_maxd)
4778
4779	DO_ZPZZ_FP(sve_fminnum_b16, uint16_t, H1_2, bfloat16_minnum)
4780	DO_ZPZZ_FP(sve_fminnum_h, uint16_t, H1_2, float16_minnum)
4781	DO_ZPZZ_FP(sve_fminnum_s, uint32_t, H1_4, float32_minnum)
4782	DO_ZPZZ_FP(sve_fminnum_d, uint64_t, H1_8, float64_minnum)
4783
4784	DO_ZPZZ_FP(sve_fmaxnum_b16, uint16_t, H1_2, bfloat16_maxnum)
4785	DO_ZPZZ_FP(sve_fmaxnum_h, uint16_t, H1_2, float16_maxnum)
4786	DO_ZPZZ_FP(sve_fmaxnum_s, uint32_t, H1_4, float32_maxnum)
4787	DO_ZPZZ_FP(sve_fmaxnum_d, uint64_t, H1_8, float64_maxnum)
4788
4789	static inline float16 abd_h(float16 a, float16 b, float_status *s)
4790	{
4791	return float16_abs(float16_sub(a, b, s));
4792	}
4793
4794	static inline float32 abd_s(float32 a, float32 b, float_status *s)
4795	{
4796	return float32_abs(float32_sub(a, b, s));
4797	}
4798
4799	static inline float64 abd_d(float64 a, float64 b, float_status *s)
4800	{
4801	return float64_abs(float64_sub(a, b, s));
4802	}
4803
4804	/* ABD when FPCR.AH = 1: avoid flipping sign bit of a NaN result */
4805	static float16 ah_abd_h(float16 op1, float16 op2, float_status *stat)
4806	{
4807	float16 r = float16_sub(op1, op2, stat);
4808	return float16_is_any_nan(r) ? r : float16_abs(r);
4809	}
4810
4811	static float32 ah_abd_s(float32 op1, float32 op2, float_status *stat)
4812	{
4813	float32 r = float32_sub(op1, op2, stat);
4814	return float32_is_any_nan(r) ? r : float32_abs(r);
4815	}
4816
4817	static float64 ah_abd_d(float64 op1, float64 op2, float_status *stat)
4818	{
4819	float64 r = float64_sub(op1, op2, stat);
4820	return float64_is_any_nan(r) ? r : float64_abs(r);
4821	}
4822
4823	DO_ZPZZ_FP(sve_fabd_h, uint16_t, H1_2, abd_h)
4824	DO_ZPZZ_FP(sve_fabd_s, uint32_t, H1_4, abd_s)
4825	DO_ZPZZ_FP(sve_fabd_d, uint64_t, H1_8, abd_d)
4826	DO_ZPZZ_FP(sve_ah_fabd_h, uint16_t, H1_2, ah_abd_h)
4827	DO_ZPZZ_FP(sve_ah_fabd_s, uint32_t, H1_4, ah_abd_s)
4828	DO_ZPZZ_FP(sve_ah_fabd_d, uint64_t, H1_8, ah_abd_d)
4829
4830	DO_ZPZZ_FP(sve_fscalbn_b16, int16_t, H1_2, bfloat16_scalbn)
4831	DO_ZPZZ_FP(sve_fscalbn_h, int16_t, H1_2, float16_scalbn)
4832	DO_ZPZZ_FP(sve_fscalbn_s, int32_t, H1_4, float32_scalbn)
4833	DO_ZPZZ_FP(sve_fscalbn_d, int64_t, H1_8, scalbn_d)
4834
4835	DO_ZPZZ_FP(sve_fmulx_h, uint16_t, H1_2, helper_advsimd_mulxh)
4836	DO_ZPZZ_FP(sve_fmulx_s, uint32_t, H1_4, helper_vfp_mulxs)
4837	DO_ZPZZ_FP(sve_fmulx_d, uint64_t, H1_8, helper_vfp_mulxd)
4838
4839	DO_ZPZZ_FP(sve2_famax_h, uint16_t, H1_2, float16_famax)
4840	DO_ZPZZ_FP(sve2_famax_s, uint32_t, H1_4, float32_famax)
4841	DO_ZPZZ_FP(sve2_famax_d, uint64_t, H1_8, float64_famax)
4842
4843	DO_ZPZZ_FP(sve2_famin_h, uint16_t, H1_2, float16_famin)
4844	DO_ZPZZ_FP(sve2_famin_s, uint32_t, H1_4, float32_famin)
4845	DO_ZPZZ_FP(sve2_famin_d, uint64_t, H1_8, float64_famin)
4846
4847	#undef DO_ZPZZ_FP
4848
4849	/* Three-operand expander, with one scalar operand, controlled by
4850	* a predicate, with the extra float_status parameter.
4851	*/
4852	#define DO_ZPZS_FP(NAME, TYPE, H, OP) \
4853	void HELPER(NAME)(void vd, void vn, void *vg, uint64_t scalar, \
4854	float_status *status, uint32_t desc) \
4855	{ \
4856	intptr_t i = simd_oprsz(desc); \
4857	uint64_t *g = vg; \
4858	TYPE mm = scalar; \
4859	do { \
4860	uint64_t pg = g[(i - 1) >> 6]; \
4861	do { \
4862	i -= sizeof(TYPE); \
4863	if (likely((pg >> (i & 63)) & 1)) { \
4864	TYPE nn = (TYPE )(vn + H(i)); \
4865	(TYPE )(vd + H(i)) = OP(nn, mm, status); \
4866	} \
4867	} while (i & 63); \
4868	} while (i != 0); \
4869	}
4870
4871	DO_ZPZS_FP(sve_fadds_h, float16, H1_2, float16_add)
4872	DO_ZPZS_FP(sve_fadds_s, float32, H1_4, float32_add)
4873	DO_ZPZS_FP(sve_fadds_d, float64, H1_8, float64_add)
4874
4875	DO_ZPZS_FP(sve_fsubs_h, float16, H1_2, float16_sub)
4876	DO_ZPZS_FP(sve_fsubs_s, float32, H1_4, float32_sub)
4877	DO_ZPZS_FP(sve_fsubs_d, float64, H1_8, float64_sub)
4878
4879	DO_ZPZS_FP(sve_fmuls_h, float16, H1_2, float16_mul)
4880	DO_ZPZS_FP(sve_fmuls_s, float32, H1_4, float32_mul)
4881	DO_ZPZS_FP(sve_fmuls_d, float64, H1_8, float64_mul)
4882
4883	static inline float16 subr_h(float16 a, float16 b, float_status *s)
4884	{
4885	return float16_sub(b, a, s);
4886	}
4887
4888	static inline float32 subr_s(float32 a, float32 b, float_status *s)
4889	{
4890	return float32_sub(b, a, s);
4891	}
4892
4893	static inline float64 subr_d(float64 a, float64 b, float_status *s)
4894	{
4895	return float64_sub(b, a, s);
4896	}
4897
4898	DO_ZPZS_FP(sve_fsubrs_h, float16, H1_2, subr_h)
4899	DO_ZPZS_FP(sve_fsubrs_s, float32, H1_4, subr_s)
4900	DO_ZPZS_FP(sve_fsubrs_d, float64, H1_8, subr_d)
4901
4902	DO_ZPZS_FP(sve_fmaxnms_h, float16, H1_2, float16_maxnum)
4903	DO_ZPZS_FP(sve_fmaxnms_s, float32, H1_4, float32_maxnum)
4904	DO_ZPZS_FP(sve_fmaxnms_d, float64, H1_8, float64_maxnum)
4905
4906	DO_ZPZS_FP(sve_fminnms_h, float16, H1_2, float16_minnum)
4907	DO_ZPZS_FP(sve_fminnms_s, float32, H1_4, float32_minnum)
4908	DO_ZPZS_FP(sve_fminnms_d, float64, H1_8, float64_minnum)
4909
4910	DO_ZPZS_FP(sve_fmaxs_h, float16, H1_2, float16_max)
4911	DO_ZPZS_FP(sve_fmaxs_s, float32, H1_4, float32_max)
4912	DO_ZPZS_FP(sve_fmaxs_d, float64, H1_8, float64_max)
4913
4914	DO_ZPZS_FP(sve_fmins_h, float16, H1_2, float16_min)
4915	DO_ZPZS_FP(sve_fmins_s, float32, H1_4, float32_min)
4916	DO_ZPZS_FP(sve_fmins_d, float64, H1_8, float64_min)
4917
4918	DO_ZPZS_FP(sve_ah_fmaxs_h, float16, H1_2, helper_vfp_ah_maxh)
4919	DO_ZPZS_FP(sve_ah_fmaxs_s, float32, H1_4, helper_vfp_ah_maxs)
4920	DO_ZPZS_FP(sve_ah_fmaxs_d, float64, H1_8, helper_vfp_ah_maxd)
4921
4922	DO_ZPZS_FP(sve_ah_fmins_h, float16, H1_2, helper_vfp_ah_minh)
4923	DO_ZPZS_FP(sve_ah_fmins_s, float32, H1_4, helper_vfp_ah_mins)
4924	DO_ZPZS_FP(sve_ah_fmins_d, float64, H1_8, helper_vfp_ah_mind)
4925
4926	/*
4927	* Fully general two-operand expander, controlled by a predicate,
4928	* With the extra float_status parameter.
4929	*/
4930	#define DO_ZPZ_FP(NAME, TYPE, H, OP) \
4931	void HELPER(NAME)(void vd, void vn, void *vg, \
4932	float_status *status, uint32_t desc) \
4933	{ \
4934	intptr_t i = simd_oprsz(desc); \
4935	bool zeroing = simd_data(desc) & 1; \
4936	uint64_t *g = vg; \
4937	do { \
4938	uint64_t pg = g[(i - 1) >> 6]; \
4939	do { \
4940	i -= sizeof(TYPE); \
4941	if (likely((pg >> (i & 63)) & 1)) { \
4942	TYPE nn = (TYPE )(vn + H(i)); \
4943	(TYPE )(vd + H(i)) = OP(nn, status); \
4944	} else if (zeroing) { \
4945	(TYPE )(vd + H(i)) = 0; \
4946	} \
4947	} while (i & 63); \
4948	} while (i != 0); \
4949	}
4950
4951	/* SVE fp16 conversions always use IEEE mode. Like AdvSIMD, they ignore
4952	* FZ16. When converting from fp16, this affects flushing input denormals;
4953	* when converting to fp16, this affects flushing output denormals.
4954	*/
4955	float32 sve_f16_to_f32(float16 f, float_status *fpst)
4956	{
4957	bool save = get_flush_inputs_to_zero(fpst);
4958	float32 ret;
4959
4960	set_flush_inputs_to_zero(false, fpst);
4961	ret = float16_to_float32(f, true, fpst);
4962	set_flush_inputs_to_zero(save, fpst);
4963	return ret;
4964	}
4965
4966	static inline float64 sve_f16_to_f64(float16 f, float_status *fpst)
4967	{
4968	bool save = get_flush_inputs_to_zero(fpst);
4969	float64 ret;
4970
4971	set_flush_inputs_to_zero(false, fpst);
4972	ret = float16_to_float64(f, true, fpst);
4973	set_flush_inputs_to_zero(save, fpst);
4974	return ret;
4975	}
4976
4977	float16 sve_f32_to_f16(float32 f, float_status *fpst)
4978	{
4979	bool save = get_flush_to_zero(fpst);
4980	float16 ret;
4981
4982	set_flush_to_zero(false, fpst);
4983	ret = float32_to_float16(f, true, fpst);
4984	set_flush_to_zero(save, fpst);
4985	return ret;
4986	}
4987
4988	static inline float16 sve_f64_to_f16(float64 f, float_status *fpst)
4989	{
4990	bool save = get_flush_to_zero(fpst);
4991	float16 ret;
4992
4993	set_flush_to_zero(false, fpst);
4994	ret = float64_to_float16(f, true, fpst);
4995	set_flush_to_zero(save, fpst);
4996	return ret;
4997	}
4998
4999	static inline int16_t vfp_float16_to_int16_rtz(float16 f, float_status *s)
5000	{

Showing first 5,000 of 8,754 lines. View raw