mve_helper.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 3,472 lines 166 KB

Raw

1	/*
2	* M-profile MVE Operations
3	*
4	* Copyright (c) 2021 Linaro, Ltd.
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#include "qemu/osdep.h"
21	#include "cpu.h"
22	#include "helper.h"
23	#include "helper-mve.h"
24	#include "internals.h"
25	#include "vec_internal.h"
26	#include "accel/tcg/cpu-ldst-common.h"
27	#include "tcg/tcg.h"
28	#include "fpu/softfloat.h"
29	#include "crypto/clmul.h"
30
31	#define HELPER_H "tcg/helper-mve-defs.h"
32	#include "exec/helper-info.c.inc"
33
34	static uint16_t mve_eci_mask(CPUARMState *env)
35	{
36	/*
37	* Return the mask of which elements in the MVE vector correspond
38	* to beats being executed. The mask has 1 bits for executed lanes
39	* and 0 bits where ECI says this beat was already executed.
40	*/
41	int eci;
42
43	if ((env->condexec_bits & 0xf) != 0) {
44	return 0xffff;
45	}
46
47	eci = env->condexec_bits >> 4;
48	switch (eci) {
49	case ECI_NONE:
50	return 0xffff;
51	case ECI_A0:
52	return 0xfff0;
53	case ECI_A0A1:
54	return 0xff00;
55	case ECI_A0A1A2:
56	case ECI_A0A1A2B0:
57	return 0xf000;
58	default:
59	g_assert_not_reached();
60	}
61	}
62
63	static uint16_t mve_element_mask(CPUARMState *env)
64	{
65	/*
66	* Return the mask of which elements in the MVE vector should be
67	* updated. This is a combination of multiple things:
68	* (1) by default, we update every lane in the vector
69	* (2) VPT predication stores its state in the VPR register;
70	* (3) low-overhead-branch tail predication will mask out part
71	* the vector on the final iteration of the loop
72	* (4) if EPSR.ECI is set then we must execute only some beats
73	* of the insn
74	* We combine all these into a 16-bit result with the same semantics
75	* as VPR.P0: 0 to mask the lane, 1 if it is active.
76	* 8-bit vector ops will look at all bits of the result;
77	* 16-bit ops will look at bits 0, 2, 4, ...;
78	* 32-bit ops will look at bits 0, 4, 8 and 12.
79	* Compare pseudocode GetCurInstrBeat(), though that only returns
80	* the 4-bit slice of the mask corresponding to a single beat.
81	*/
82	uint16_t mask = FIELD_EX32(env->v7m.vpr, V7M_VPR, P0);
83
84	if (!(env->v7m.vpr & R_V7M_VPR_MASK01_MASK)) {
85	mask \|= 0xff;
86	}
87	if (!(env->v7m.vpr & R_V7M_VPR_MASK23_MASK)) {
88	mask \|= 0xff00;
89	}
90
91	if (env->v7m.ltpsize < 4 &&
92	env->regs[14] <= (1 << (4 - env->v7m.ltpsize))) {
93	/*
94	* Tail predication active, and this is the last loop iteration.
95	* The element size is (1 << ltpsize), and we only want to process
96	* loopcount elements, so we want to retain the least significant
97	* (loopcount * esize) predicate bits and zero out bits above that.
98	*/
99	int masklen = env->regs[14] << env->v7m.ltpsize;
100	assert(masklen <= 16);
101	uint16_t ltpmask = masklen ? MAKE_64BIT_MASK(0, masklen) : 0;
102	mask &= ltpmask;
103	}
104
105	/*
106	* ECI bits indicate which beats are already executed;
107	* we handle this by effectively predicating them out.
108	*/
109	mask &= mve_eci_mask(env);
110	return mask;
111	}
112
113	static void mve_advance_vpt(CPUARMState *env)
114	{
115	/* Advance the VPT and ECI state if necessary */
116	uint32_t vpr = env->v7m.vpr;
117	unsigned mask01, mask23;
118	uint16_t inv_mask;
119	uint16_t eci_mask = mve_eci_mask(env);
120
121	if ((env->condexec_bits & 0xf) == 0) {
122	env->condexec_bits = (env->condexec_bits == (ECI_A0A1A2B0 << 4)) ?
123	(ECI_A0 << 4) : (ECI_NONE << 4);
124	}
125
126	if (!(vpr & (R_V7M_VPR_MASK01_MASK \| R_V7M_VPR_MASK23_MASK))) {
127	/* VPT not enabled, nothing to do */
128	return;
129	}
130
131	/* Invert P0 bits if needed, but only for beats we actually executed */
132	mask01 = FIELD_EX32(vpr, V7M_VPR, MASK01);
133	mask23 = FIELD_EX32(vpr, V7M_VPR, MASK23);
134	/* Start by assuming we invert all bits corresponding to executed beats */
135	inv_mask = eci_mask;
136	if (mask01 <= 8) {
137	/* MASK01 says don't invert low half of P0 */
138	inv_mask &= ~0xff;
139	}
140	if (mask23 <= 8) {
141	/* MASK23 says don't invert high half of P0 */
142	inv_mask &= ~0xff00;
143	}
144	vpr ^= inv_mask;
145	/* Only update MASK01 if beat 1 executed */
146	if (eci_mask & 0xf0) {
147	vpr = FIELD_DP32(vpr, V7M_VPR, MASK01, mask01 << 1);
148	}
149	/* Beat 3 always executes, so update MASK23 */
150	vpr = FIELD_DP32(vpr, V7M_VPR, MASK23, mask23 << 1);
151	env->v7m.vpr = vpr;
152	}
153
154	/* For loads, predicated lanes are zeroed instead of keeping their old values */
155	#define DO_VLDR(OP, MFLAG, MSIZE, MTYPE, LDTYPE, ESIZE, TYPE) \
156	void HELPER(mve_##OP)(CPUARMState env, void vd, uint32_t addr) \
157	{ \
158	TYPE *d = vd; \
159	uint16_t mask = mve_element_mask(env); \
160	uint16_t eci_mask = mve_eci_mask(env); \
161	unsigned b, e; \
162	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
163	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MFLAG \| MO_ALIGN, \
164	mmu_idx); \
165	/* \
166	* R_SXTM allows the dest reg to become UNKNOWN for abandoned \
167	* beats so we don't care if we update part of the dest and \
168	* then take an exception. \
169	*/ \
170	for (b = 0, e = 0; b < 16; b += ESIZE, e++) { \
171	if (eci_mask & (1 << b)) { \
172	d[H##ESIZE(e)] = (mask & (1 << b)) ? \
173	(MTYPE)cpu_##LDTYPE##_mmu(env, addr, oi, GETPC()) : 0;\
174	} \
175	addr += MSIZE; \
176	} \
177	mve_advance_vpt(env); \
178	}
179
180	#define DO_VSTR(OP, MFLAG, MSIZE, STTYPE, ESIZE, TYPE) \
181	void HELPER(mve_##OP)(CPUARMState env, void vd, uint32_t addr) \
182	{ \
183	TYPE *d = vd; \
184	uint16_t mask = mve_element_mask(env); \
185	unsigned b, e; \
186	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
187	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MFLAG \| MO_ALIGN, \
188	mmu_idx); \
189	for (b = 0, e = 0; b < 16; b += ESIZE, e++) { \
190	if (mask & (1 << b)) { \
191	cpu_##STTYPE##_mmu(env, addr, d[H##ESIZE(e)], oi, GETPC()); \
192	} \
193	addr += MSIZE; \
194	} \
195	mve_advance_vpt(env); \
196	}
197
198	DO_VLDR(vldrb, MO_UB, 1, uint8_t, ldb, 1, uint8_t)
199	DO_VLDR(vldrh, MO_UW, 2, uint16_t, ldw, 2, uint16_t)
200	DO_VLDR(vldrw, MO_UL, 4, uint32_t, ldl, 4, uint32_t)
201
202	DO_VSTR(vstrb, MO_UB, 1, stb, 1, uint8_t)
203	DO_VSTR(vstrh, MO_UW, 2, stw, 2, uint16_t)
204	DO_VSTR(vstrw, MO_UL, 4, stl, 4, uint32_t)
205
206	DO_VLDR(vldrb_sh, MO_SB, 1, int8_t, ldb, 2, int16_t)
207	DO_VLDR(vldrb_sw, MO_SB, 1, int8_t, ldb, 4, int32_t)
208	DO_VLDR(vldrb_uh, MO_UB, 1, uint8_t, ldb, 2, uint16_t)
209	DO_VLDR(vldrb_uw, MO_UB, 1, uint8_t, ldb, 4, uint32_t)
210	DO_VLDR(vldrh_sw, MO_SW, 2, int16_t, ldw, 4, int32_t)
211	DO_VLDR(vldrh_uw, MO_UW, 2, uint16_t, ldw, 4, uint32_t)
212
213	DO_VSTR(vstrb_h, MO_UB, 1, stb, 2, int16_t)
214	DO_VSTR(vstrb_w, MO_UB, 1, stb, 4, int32_t)
215	DO_VSTR(vstrh_w, MO_UW, 2, stw, 4, int32_t)
216
217	#undef DO_VLDR
218	#undef DO_VSTR
219
220	/*
221	* Gather loads/scatter stores. Here each element of Qm specifies
222	* an offset to use from the base register Rm. In the _os_ versions
223	* that offset is scaled by the element size.
224	* For loads, predicated lanes are zeroed instead of retaining
225	* their previous values.
226	*/
227	#define DO_VLDR_SG(OP, MFLAG, MTYPE, LDTYPE, ESIZE, TYPE, OFFTYPE, ADDRFN, WB)\
228	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm, \
229	uint32_t base) \
230	{ \
231	TYPE *d = vd; \
232	OFFTYPE *m = vm; \
233	uint16_t mask = mve_element_mask(env); \
234	uint16_t eci_mask = mve_eci_mask(env); \
235	unsigned e; \
236	uint32_t addr; \
237	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
238	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MFLAG \| MO_ALIGN, \
239	mmu_idx); \
240	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE, eci_mask >>= ESIZE) { \
241	if (!(eci_mask & 1)) { \
242	continue; \
243	} \
244	addr = ADDRFN(base, m[H##ESIZE(e)]); \
245	d[H##ESIZE(e)] = (mask & 1) ? \
246	(MTYPE)cpu_##LDTYPE##_mmu(env, addr, oi, GETPC()) : 0; \
247	if (WB) { \
248	m[H##ESIZE(e)] = addr; \
249	} \
250	} \
251	mve_advance_vpt(env); \
252	}
253
254	/* We know here TYPE is unsigned so always the same as the offset type */
255	#define DO_VSTR_SG(OP, MFLAG, STTYPE, ESIZE, TYPE, ADDRFN, WB) \
256	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm, \
257	uint32_t base) \
258	{ \
259	TYPE *d = vd; \
260	TYPE *m = vm; \
261	uint16_t mask = mve_element_mask(env); \
262	uint16_t eci_mask = mve_eci_mask(env); \
263	unsigned e; \
264	uint32_t addr; \
265	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
266	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MFLAG \| MO_ALIGN, \
267	mmu_idx); \
268	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE, eci_mask >>= ESIZE) { \
269	if (!(eci_mask & 1)) { \
270	continue; \
271	} \
272	addr = ADDRFN(base, m[H##ESIZE(e)]); \
273	if (mask & 1) { \
274	cpu_##STTYPE##_mmu(env, addr, d[H##ESIZE(e)], oi, GETPC()); \
275	} \
276	if (WB) { \
277	m[H##ESIZE(e)] = addr; \
278	} \
279	} \
280	mve_advance_vpt(env); \
281	}
282
283	/*
284	* 64-bit accesses are slightly different: they are done as two 32-bit
285	* accesses, controlled by the predicate mask for the relevant beat,
286	* and with a single 32-bit offset in the first of the two Qm elements.
287	* Note that for QEMU our IMPDEF AIRCR.ENDIANNESS is always 0 (little).
288	* Address writeback happens on the odd beats and updates the address
289	* stored in the even-beat element.
290	*/
291	#define DO_VLDR64_SG(OP, ADDRFN, WB) \
292	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm, \
293	uint32_t base) \
294	{ \
295	uint32_t *d = vd; \
296	uint32_t *m = vm; \
297	uint16_t mask = mve_element_mask(env); \
298	uint16_t eci_mask = mve_eci_mask(env); \
299	unsigned e; \
300	uint32_t addr; \
301	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
302	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
303	mmu_idx); \
304	for (e = 0; e < 16 / 4; e++, mask >>= 4, eci_mask >>= 4) { \
305	if (!(eci_mask & 1)) { \
306	continue; \
307	} \
308	addr = ADDRFN(base, m[H4(e & ~1)]); \
309	addr += 4 * (e & 1); \
310	d[H4(e)] = (mask & 1) ? cpu_ldl_mmu(env, addr, oi, GETPC()) : 0; \
311	if (WB && (e & 1)) { \
312	m[H4(e & ~1)] = addr - 4; \
313	} \
314	} \
315	mve_advance_vpt(env); \
316	}
317
318	#define DO_VSTR64_SG(OP, ADDRFN, WB) \
319	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm, \
320	uint32_t base) \
321	{ \
322	uint32_t *d = vd; \
323	uint32_t *m = vm; \
324	uint16_t mask = mve_element_mask(env); \
325	uint16_t eci_mask = mve_eci_mask(env); \
326	unsigned e; \
327	uint32_t addr; \
328	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
329	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
330	mmu_idx); \
331	for (e = 0; e < 16 / 4; e++, mask >>= 4, eci_mask >>= 4) { \
332	if (!(eci_mask & 1)) { \
333	continue; \
334	} \
335	addr = ADDRFN(base, m[H4(e & ~1)]); \
336	addr += 4 * (e & 1); \
337	if (mask & 1) { \
338	cpu_stl_mmu(env, addr, d[H4(e)], oi, GETPC()); \
339	} \
340	if (WB && (e & 1)) { \
341	m[H4(e & ~1)] = addr - 4; \
342	} \
343	} \
344	mve_advance_vpt(env); \
345	}
346
347	#define ADDR_ADD(BASE, OFFSET) ((BASE) + (OFFSET))
348	#define ADDR_ADD_OSH(BASE, OFFSET) ((BASE) + ((OFFSET) << 1))
349	#define ADDR_ADD_OSW(BASE, OFFSET) ((BASE) + ((OFFSET) << 2))
350	#define ADDR_ADD_OSD(BASE, OFFSET) ((BASE) + ((OFFSET) << 3))
351
352	DO_VLDR_SG(vldrb_sg_sh, MO_SB, int8_t, ldb, 2, int16_t, uint16_t, ADDR_ADD, false)
353	DO_VLDR_SG(vldrb_sg_sw, MO_SB, int8_t, ldb, 4, int32_t, uint32_t, ADDR_ADD, false)
354	DO_VLDR_SG(vldrh_sg_sw, MO_SW, int16_t, ldw, 4, int32_t, uint32_t, ADDR_ADD, false)
355
356	DO_VLDR_SG(vldrb_sg_ub, MO_UB, uint8_t, ldb, 1, uint8_t, uint8_t, ADDR_ADD, false)
357	DO_VLDR_SG(vldrb_sg_uh, MO_UB, uint8_t, ldb, 2, uint16_t, uint16_t, ADDR_ADD, false)
358	DO_VLDR_SG(vldrb_sg_uw, MO_UB, uint8_t, ldb, 4, uint32_t, uint32_t, ADDR_ADD, false)
359	DO_VLDR_SG(vldrh_sg_uh, MO_UW, uint16_t, ldw, 2, uint16_t, uint16_t, ADDR_ADD, false)
360	DO_VLDR_SG(vldrh_sg_uw, MO_UW, uint16_t, ldw, 4, uint32_t, uint32_t, ADDR_ADD, false)
361	DO_VLDR_SG(vldrw_sg_uw, MO_UL, uint32_t, ldl, 4, uint32_t, uint32_t, ADDR_ADD, false)
362	DO_VLDR64_SG(vldrd_sg_ud, ADDR_ADD, false)
363
364	DO_VLDR_SG(vldrh_sg_os_sw, MO_SW, int16_t, ldw, 4,
365	int32_t, uint32_t, ADDR_ADD_OSH, false)
366	DO_VLDR_SG(vldrh_sg_os_uh, MO_UW, uint16_t, ldw, 2,
367	uint16_t, uint16_t, ADDR_ADD_OSH, false)
368	DO_VLDR_SG(vldrh_sg_os_uw, MO_UW, uint16_t, ldw, 4,
369	uint32_t, uint32_t, ADDR_ADD_OSH, false)
370	DO_VLDR_SG(vldrw_sg_os_uw, MO_UL, uint32_t, ldl, 4,
371	uint32_t, uint32_t, ADDR_ADD_OSW, false)
372	DO_VLDR64_SG(vldrd_sg_os_ud, ADDR_ADD_OSD, false)
373
374	DO_VSTR_SG(vstrb_sg_ub, MO_UB, stb, 1, uint8_t, ADDR_ADD, false)
375	DO_VSTR_SG(vstrb_sg_uh, MO_UB, stb, 2, uint16_t, ADDR_ADD, false)
376	DO_VSTR_SG(vstrb_sg_uw, MO_UB, stb, 4, uint32_t, ADDR_ADD, false)
377	DO_VSTR_SG(vstrh_sg_uh, MO_UW, stw, 2, uint16_t, ADDR_ADD, false)
378	DO_VSTR_SG(vstrh_sg_uw, MO_UW, stw, 4, uint32_t, ADDR_ADD, false)
379	DO_VSTR_SG(vstrw_sg_uw, MO_UL, stl, 4, uint32_t, ADDR_ADD, false)
380	DO_VSTR64_SG(vstrd_sg_ud, ADDR_ADD, false)
381
382	DO_VSTR_SG(vstrh_sg_os_uh, MO_UW, stw, 2, uint16_t, ADDR_ADD_OSH, false)
383	DO_VSTR_SG(vstrh_sg_os_uw, MO_UW, stw, 4, uint32_t, ADDR_ADD_OSH, false)
384	DO_VSTR_SG(vstrw_sg_os_uw, MO_UL, stl, 4, uint32_t, ADDR_ADD_OSW, false)
385	DO_VSTR64_SG(vstrd_sg_os_ud, ADDR_ADD_OSD, false)
386
387	DO_VLDR_SG(vldrw_sg_wb_uw, MO_UL, uint32_t, ldl, 4, uint32_t, uint32_t, ADDR_ADD, true)
388	DO_VLDR64_SG(vldrd_sg_wb_ud, ADDR_ADD, true)
389	DO_VSTR_SG(vstrw_sg_wb_uw, MO_UL, stl, 4, uint32_t, ADDR_ADD, true)
390	DO_VSTR64_SG(vstrd_sg_wb_ud, ADDR_ADD, true)
391
392	/*
393	* Deinterleaving loads/interleaving stores.
394	*
395	* For these helpers we are passed the index of the first Qreg
396	* (VLD2/VST2 will also access Qn+1, VLD4/VST4 access Qn .. Qn+3)
397	* and the value of the base address register Rn.
398	* The helpers are specialized for pattern and element size, so
399	* for instance vld42h is VLD4 with pattern 2, element size MO_16.
400	*
401	* These insns are beatwise but not predicated, so we must honour ECI,
402	* but need not look at mve_element_mask().
403	*
404	* The pseudocode implements these insns with multiple memory accesses
405	* of the element size, but rules R_VVVG and R_FXDM permit us to make
406	* one 32-bit memory access per beat.
407	*/
408	#define DO_VLD4B(OP, O1, O2, O3, O4) \
409	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
410	uint32_t base) \
411	{ \
412	int beat, e; \
413	uint16_t mask = mve_eci_mask(env); \
414	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
415	uint32_t addr, data; \
416	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
417	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
418	mmu_idx); \
419	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
420	if ((mask & 1) == 0) { \
421	/* ECI says skip this beat */ \
422	continue; \
423	} \
424	addr = base + off[beat] * 4; \
425	data = cpu_ldl_mmu(env, addr, oi, GETPC()); \
426	for (e = 0; e < 4; e++, data >>= 8) { \
427	uint8_t qd = (uint8_t )aa32_vfp_qreg(env, qnidx + e); \
428	qd[H1(off[beat])] = data; \
429	} \
430	} \
431	}
432
433	#define DO_VLD4H(OP, O1, O2) \
434	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
435	uint32_t base) \
436	{ \
437	int beat; \
438	uint16_t mask = mve_eci_mask(env); \
439	static const uint8_t off[4] = { O1, O1, O2, O2 }; \
440	uint32_t addr, data; \
441	int y; /* y counts 0 2 0 2 */ \
442	uint16_t *qd; \
443	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
444	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
445	mmu_idx); \
446	for (beat = 0, y = 0; beat < 4; beat++, mask >>= 4, y ^= 2) { \
447	if ((mask & 1) == 0) { \
448	/* ECI says skip this beat */ \
449	continue; \
450	} \
451	addr = base + off[beat] * 8 + (beat & 1) * 4; \
452	data = cpu_ldl_mmu(env, addr, oi, GETPC()); \
453	qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + y); \
454	qd[H2(off[beat])] = data; \
455	data >>= 16; \
456	qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + y + 1); \
457	qd[H2(off[beat])] = data; \
458	} \
459	}
460
461	#define DO_VLD4W(OP, O1, O2, O3, O4) \
462	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
463	uint32_t base) \
464	{ \
465	int beat; \
466	uint16_t mask = mve_eci_mask(env); \
467	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
468	uint32_t addr, data; \
469	uint32_t *qd; \
470	int y; \
471	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
472	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
473	mmu_idx); \
474	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
475	if ((mask & 1) == 0) { \
476	/* ECI says skip this beat */ \
477	continue; \
478	} \
479	addr = base + off[beat] * 4; \
480	data = cpu_ldl_mmu(env, addr, oi, GETPC()); \
481	y = (beat + (O1 & 2)) & 3; \
482	qd = (uint32_t *)aa32_vfp_qreg(env, qnidx + y); \
483	qd[H4(off[beat] >> 2)] = data; \
484	} \
485	}
486
487	DO_VLD4B(vld40b, 0, 1, 10, 11)
488	DO_VLD4B(vld41b, 2, 3, 12, 13)
489	DO_VLD4B(vld42b, 4, 5, 14, 15)
490	DO_VLD4B(vld43b, 6, 7, 8, 9)
491
492	DO_VLD4H(vld40h, 0, 5)
493	DO_VLD4H(vld41h, 1, 6)
494	DO_VLD4H(vld42h, 2, 7)
495	DO_VLD4H(vld43h, 3, 4)
496
497	DO_VLD4W(vld40w, 0, 1, 10, 11)
498	DO_VLD4W(vld41w, 2, 3, 12, 13)
499	DO_VLD4W(vld42w, 4, 5, 14, 15)
500	DO_VLD4W(vld43w, 6, 7, 8, 9)
501
502	#define DO_VLD2B(OP, O1, O2, O3, O4) \
503	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
504	uint32_t base) \
505	{ \
506	int beat, e; \
507	uint16_t mask = mve_eci_mask(env); \
508	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
509	uint32_t addr, data; \
510	uint8_t *qd; \
511	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
512	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
513	mmu_idx); \
514	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
515	if ((mask & 1) == 0) { \
516	/* ECI says skip this beat */ \
517	continue; \
518	} \
519	addr = base + off[beat] * 2; \
520	data = cpu_ldl_mmu(env, addr, oi, GETPC()); \
521	for (e = 0; e < 4; e++, data >>= 8) { \
522	qd = (uint8_t *)aa32_vfp_qreg(env, qnidx + (e & 1)); \
523	qd[H1(off[beat] + (e >> 1))] = data; \
524	} \
525	} \
526	}
527
528	#define DO_VLD2H(OP, O1, O2, O3, O4) \
529	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
530	uint32_t base) \
531	{ \
532	int beat; \
533	uint16_t mask = mve_eci_mask(env); \
534	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
535	uint32_t addr, data; \
536	int e; \
537	uint16_t *qd; \
538	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
539	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
540	mmu_idx); \
541	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
542	if ((mask & 1) == 0) { \
543	/* ECI says skip this beat */ \
544	continue; \
545	} \
546	addr = base + off[beat] * 4; \
547	data = cpu_ldl_mmu(env, addr, oi, GETPC()); \
548	for (e = 0; e < 2; e++, data >>= 16) { \
549	qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + e); \
550	qd[H2(off[beat])] = data; \
551	} \
552	} \
553	}
554
555	#define DO_VLD2W(OP, O1, O2, O3, O4) \
556	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
557	uint32_t base) \
558	{ \
559	int beat; \
560	uint16_t mask = mve_eci_mask(env); \
561	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
562	uint32_t addr, data; \
563	uint32_t *qd; \
564	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
565	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
566	mmu_idx); \
567	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
568	if ((mask & 1) == 0) { \
569	/* ECI says skip this beat */ \
570	continue; \
571	} \
572	addr = base + off[beat]; \
573	data = cpu_ldl_mmu(env, addr, oi, GETPC()); \
574	qd = (uint32_t *)aa32_vfp_qreg(env, qnidx + (beat & 1)); \
575	qd[H4(off[beat] >> 3)] = data; \
576	} \
577	}
578
579	DO_VLD2B(vld20b, 0, 2, 12, 14)
580	DO_VLD2B(vld21b, 4, 6, 8, 10)
581
582	DO_VLD2H(vld20h, 0, 1, 6, 7)
583	DO_VLD2H(vld21h, 2, 3, 4, 5)
584
585	DO_VLD2W(vld20w, 0, 4, 24, 28)
586	DO_VLD2W(vld21w, 8, 12, 16, 20)
587
588	#define DO_VST4B(OP, O1, O2, O3, O4) \
589	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
590	uint32_t base) \
591	{ \
592	int beat, e; \
593	uint16_t mask = mve_eci_mask(env); \
594	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
595	uint32_t addr, data; \
596	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
597	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
598	mmu_idx); \
599	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
600	if ((mask & 1) == 0) { \
601	/* ECI says skip this beat */ \
602	continue; \
603	} \
604	addr = base + off[beat] * 4; \
605	data = 0; \
606	for (e = 3; e >= 0; e--) { \
607	uint8_t qd = (uint8_t )aa32_vfp_qreg(env, qnidx + e); \
608	data = (data << 8) \| qd[H1(off[beat])]; \
609	} \
610	cpu_stl_mmu(env, addr, data, oi, GETPC()); \
611	} \
612	}
613
614	#define DO_VST4H(OP, O1, O2) \
615	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
616	uint32_t base) \
617	{ \
618	int beat; \
619	uint16_t mask = mve_eci_mask(env); \
620	static const uint8_t off[4] = { O1, O1, O2, O2 }; \
621	uint32_t addr, data; \
622	int y; /* y counts 0 2 0 2 */ \
623	uint16_t *qd; \
624	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
625	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
626	mmu_idx); \
627	for (beat = 0, y = 0; beat < 4; beat++, mask >>= 4, y ^= 2) { \
628	if ((mask & 1) == 0) { \
629	/* ECI says skip this beat */ \
630	continue; \
631	} \
632	addr = base + off[beat] * 8 + (beat & 1) * 4; \
633	qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + y); \
634	data = qd[H2(off[beat])]; \
635	qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + y + 1); \
636	data \|= qd[H2(off[beat])] << 16; \
637	cpu_stl_mmu(env, addr, data, oi, GETPC()); \
638	} \
639	}
640
641	#define DO_VST4W(OP, O1, O2, O3, O4) \
642	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
643	uint32_t base) \
644	{ \
645	int beat; \
646	uint16_t mask = mve_eci_mask(env); \
647	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
648	uint32_t addr, data; \
649	uint32_t *qd; \
650	int y; \
651	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
652	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
653	mmu_idx); \
654	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
655	if ((mask & 1) == 0) { \
656	/* ECI says skip this beat */ \
657	continue; \
658	} \
659	addr = base + off[beat] * 4; \
660	y = (beat + (O1 & 2)) & 3; \
661	qd = (uint32_t *)aa32_vfp_qreg(env, qnidx + y); \
662	data = qd[H4(off[beat] >> 2)]; \
663	cpu_stl_mmu(env, addr, data, oi, GETPC()); \
664	} \
665	}
666
667	DO_VST4B(vst40b, 0, 1, 10, 11)
668	DO_VST4B(vst41b, 2, 3, 12, 13)
669	DO_VST4B(vst42b, 4, 5, 14, 15)
670	DO_VST4B(vst43b, 6, 7, 8, 9)
671
672	DO_VST4H(vst40h, 0, 5)
673	DO_VST4H(vst41h, 1, 6)
674	DO_VST4H(vst42h, 2, 7)
675	DO_VST4H(vst43h, 3, 4)
676
677	DO_VST4W(vst40w, 0, 1, 10, 11)
678	DO_VST4W(vst41w, 2, 3, 12, 13)
679	DO_VST4W(vst42w, 4, 5, 14, 15)
680	DO_VST4W(vst43w, 6, 7, 8, 9)
681
682	#define DO_VST2B(OP, O1, O2, O3, O4) \
683	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
684	uint32_t base) \
685	{ \
686	int beat, e; \
687	uint16_t mask = mve_eci_mask(env); \
688	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
689	uint32_t addr, data; \
690	uint8_t *qd; \
691	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
692	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
693	mmu_idx); \
694	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
695	if ((mask & 1) == 0) { \
696	/* ECI says skip this beat */ \
697	continue; \
698	} \
699	addr = base + off[beat] * 2; \
700	data = 0; \
701	for (e = 3; e >= 0; e--) { \
702	qd = (uint8_t *)aa32_vfp_qreg(env, qnidx + (e & 1)); \
703	data = (data << 8) \| qd[H1(off[beat] + (e >> 1))]; \
704	} \
705	cpu_stl_mmu(env, addr, data, oi, GETPC()); \
706	} \
707	}
708
709	#define DO_VST2H(OP, O1, O2, O3, O4) \
710	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
711	uint32_t base) \
712	{ \
713	int beat; \
714	uint16_t mask = mve_eci_mask(env); \
715	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
716	uint32_t addr, data; \
717	int e; \
718	uint16_t *qd; \
719	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
720	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
721	mmu_idx); \
722	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
723	if ((mask & 1) == 0) { \
724	/* ECI says skip this beat */ \
725	continue; \
726	} \
727	addr = base + off[beat] * 4; \
728	data = 0; \
729	for (e = 1; e >= 0; e--) { \
730	qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + e); \
731	data = (data << 16) \| qd[H2(off[beat])]; \
732	} \
733	cpu_stl_mmu(env, addr, data, oi, GETPC()); \
734	} \
735	}
736
737	#define DO_VST2W(OP, O1, O2, O3, O4) \
738	void HELPER(mve_##OP)(CPUARMState *env, uint32_t qnidx, \
739	uint32_t base) \
740	{ \
741	int beat; \
742	uint16_t mask = mve_eci_mask(env); \
743	static const uint8_t off[4] = { O1, O2, O3, O4 }; \
744	uint32_t addr, data; \
745	uint32_t *qd; \
746	int mmu_idx = arm_to_core_mmu_idx(arm_mmu_idx(env)); \
747	MemOpIdx oi = make_memop_idx(mo_endian(env) \| MO_UL \| MO_ALIGN, \
748	mmu_idx); \
749	for (beat = 0; beat < 4; beat++, mask >>= 4) { \
750	if ((mask & 1) == 0) { \
751	/* ECI says skip this beat */ \
752	continue; \
753	} \
754	addr = base + off[beat]; \
755	qd = (uint32_t *)aa32_vfp_qreg(env, qnidx + (beat & 1)); \
756	data = qd[H4(off[beat] >> 3)]; \
757	cpu_stl_mmu(env, addr, data, oi, GETPC()); \
758	} \
759	}
760
761	DO_VST2B(vst20b, 0, 2, 12, 14)
762	DO_VST2B(vst21b, 4, 6, 8, 10)
763
764	DO_VST2H(vst20h, 0, 1, 6, 7)
765	DO_VST2H(vst21h, 2, 3, 4, 5)
766
767	DO_VST2W(vst20w, 0, 4, 24, 28)
768	DO_VST2W(vst21w, 8, 12, 16, 20)
769
770	/*
771	* The mergemask(D, R, M) macro performs the operation "*D = R" but
772	* storing only the bytes which correspond to 1 bits in M,
773	* leaving other bytes in *D unchanged. We use _Generic
774	* to select the correct implementation based on the type of D.
775	*/
776
777	static void mergemask_ub(uint8_t *d, uint8_t r, uint16_t mask)
778	{
779	if (mask & 1) {
780	*d = r;
781	}
782	}
783
784	static void mergemask_sb(int8_t *d, int8_t r, uint16_t mask)
785	{
786	mergemask_ub((uint8_t *)d, r, mask);
787	}
788
789	static void mergemask_uh(uint16_t *d, uint16_t r, uint16_t mask)
790	{
791	uint16_t bmask = expand_pred_b(mask);
792	d = (d & ~bmask) \| (r & bmask);
793	}
794
795	static void mergemask_sh(int16_t *d, int16_t r, uint16_t mask)
796	{
797	mergemask_uh((uint16_t *)d, r, mask);
798	}
799
800	static void mergemask_uw(uint32_t *d, uint32_t r, uint16_t mask)
801	{
802	uint32_t bmask = expand_pred_b(mask);
803	d = (d & ~bmask) \| (r & bmask);
804	}
805
806	static void mergemask_sw(int32_t *d, int32_t r, uint16_t mask)
807	{
808	mergemask_uw((uint32_t *)d, r, mask);
809	}
810
811	static void mergemask_uq(uint64_t *d, uint64_t r, uint16_t mask)
812	{
813	uint64_t bmask = expand_pred_b(mask);
814	d = (d & ~bmask) \| (r & bmask);
815	}
816
817	static void mergemask_sq(int64_t *d, int64_t r, uint16_t mask)
818	{
819	mergemask_uq((uint64_t *)d, r, mask);
820	}
821
822	#define mergemask(D, R, M) \
823	_Generic(D, \
824	uint8_t *: mergemask_ub, \
825	int8_t *: mergemask_sb, \
826	uint16_t *: mergemask_uh, \
827	int16_t *: mergemask_sh, \
828	uint32_t *: mergemask_uw, \
829	int32_t *: mergemask_sw, \
830	uint64_t *: mergemask_uq, \
831	int64_t *: mergemask_sq)(D, R, M)
832
833	void HELPER(mve_vdup)(CPUARMState env, void vd, uint32_t val)
834	{
835	/*
836	* The generated code already replicated an 8 or 16 bit constant
837	* into the 32-bit value, so we only need to write the 32-bit
838	* value to all elements of the Qreg, allowing for predication.
839	*/
840	uint32_t *d = vd;
841	uint16_t mask = mve_element_mask(env);
842	unsigned e;
843	for (e = 0; e < 16 / 4; e++, mask >>= 4) {
844	mergemask(&d[H4(e)], val, mask);
845	}
846	mve_advance_vpt(env);
847	}
848
849	#define DO_1OP(OP, ESIZE, TYPE, FN) \
850	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm) \
851	{ \
852	TYPE d = vd, m = vm; \
853	uint16_t mask = mve_element_mask(env); \
854	unsigned e; \
855	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
856	mergemask(&d[H##ESIZE(e)], FN(m[H##ESIZE(e)]), mask); \
857	} \
858	mve_advance_vpt(env); \
859	}
860
861	#define DO_CLS_B(N) (clrsb32(N) - 24)
862	#define DO_CLS_H(N) (clrsb32(N) - 16)
863
864	DO_1OP(vclsb, 1, int8_t, DO_CLS_B)
865	DO_1OP(vclsh, 2, int16_t, DO_CLS_H)
866	DO_1OP(vclsw, 4, int32_t, clrsb32)
867
868	#define DO_CLZ_B(N) (clz32(N) - 24)
869	#define DO_CLZ_H(N) (clz32(N) - 16)
870
871	DO_1OP(vclzb, 1, uint8_t, DO_CLZ_B)
872	DO_1OP(vclzh, 2, uint16_t, DO_CLZ_H)
873	DO_1OP(vclzw, 4, uint32_t, clz32)
874
875	DO_1OP(vrev16b, 2, uint16_t, bswap16)
876	DO_1OP(vrev32b, 4, uint32_t, bswap32)
877	DO_1OP(vrev32h, 4, uint32_t, hswap32)
878	DO_1OP(vrev64b, 8, uint64_t, bswap64)
879	DO_1OP(vrev64h, 8, uint64_t, hswap64)
880	DO_1OP(vrev64w, 8, uint64_t, wswap64)
881
882	#define DO_NOT(N) (~(N))
883
884	DO_1OP(vmvn, 8, uint64_t, DO_NOT)
885
886	#define DO_ABS(N) ((N) < 0 ? -(N) : (N))
887	#define DO_FABSH(N) ((N) & dup_const(MO_16, 0x7fff))
888	#define DO_FABSS(N) ((N) & dup_const(MO_32, 0x7fffffff))
889
890	DO_1OP(vabsb, 1, int8_t, DO_ABS)
891	DO_1OP(vabsh, 2, int16_t, DO_ABS)
892	DO_1OP(vabsw, 4, int32_t, DO_ABS)
893
894	/* We can do these 64 bits at a time */
895	DO_1OP(vfabsh, 8, uint64_t, DO_FABSH)
896	DO_1OP(vfabss, 8, uint64_t, DO_FABSS)
897
898	#define DO_NEG(N) (-(N))
899	#define DO_FNEGH(N) ((N) ^ dup_const(MO_16, 0x8000))
900	#define DO_FNEGS(N) ((N) ^ dup_const(MO_32, 0x80000000))
901
902	DO_1OP(vnegb, 1, int8_t, DO_NEG)
903	DO_1OP(vnegh, 2, int16_t, DO_NEG)
904	DO_1OP(vnegw, 4, int32_t, DO_NEG)
905
906	/* We can do these 64 bits at a time */
907	DO_1OP(vfnegh, 8, uint64_t, DO_FNEGH)
908	DO_1OP(vfnegs, 8, uint64_t, DO_FNEGS)
909
910	/*
911	* 1 operand immediates: Vda is destination and possibly also one source.
912	* All these insns work at 64-bit widths.
913	*/
914	#define DO_1OP_IMM(OP, FN) \
915	void HELPER(mve_##OP)(CPUARMState env, void vda, uint64_t imm) \
916	{ \
917	uint64_t *da = vda; \
918	uint16_t mask = mve_element_mask(env); \
919	unsigned e; \
920	for (e = 0; e < 16 / 8; e++, mask >>= 8) { \
921	mergemask(&da[H8(e)], FN(da[H8(e)], imm), mask); \
922	} \
923	mve_advance_vpt(env); \
924	}
925
926	#define DO_MOVI(N, I) (I)
927	#define DO_ANDI(N, I) ((N) & (I))
928	#define DO_ORRI(N, I) ((N) \| (I))
929
930	DO_1OP_IMM(vmovi, DO_MOVI)
931	DO_1OP_IMM(vandi, DO_ANDI)
932	DO_1OP_IMM(vorri, DO_ORRI)
933
934	#define DO_2OP(OP, ESIZE, TYPE, FN) \
935	void HELPER(glue(mve_, OP))(CPUARMState *env, \
936	void vd, void vn, void *vm) \
937	{ \
938	TYPE d = vd, n = vn, *m = vm; \
939	uint16_t mask = mve_element_mask(env); \
940	unsigned e; \
941	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
942	mergemask(&d[H##ESIZE(e)], \
943	FN(n[H##ESIZE(e)], m[H##ESIZE(e)]), mask); \
944	} \
945	mve_advance_vpt(env); \
946	}
947
948	/* provide unsigned 2-op helpers for all sizes */
949	#define DO_2OP_U(OP, FN) \
950	DO_2OP(OP##b, 1, uint8_t, FN) \
951	DO_2OP(OP##h, 2, uint16_t, FN) \
952	DO_2OP(OP##w, 4, uint32_t, FN)
953
954	/* provide signed 2-op helpers for all sizes */
955	#define DO_2OP_S(OP, FN) \
956	DO_2OP(OP##b, 1, int8_t, FN) \
957	DO_2OP(OP##h, 2, int16_t, FN) \
958	DO_2OP(OP##w, 4, int32_t, FN)
959
960	/*
961	* "Long" operations where two half-sized inputs (taken from either the
962	* top or the bottom of the input vector) produce a double-width result.
963	* Here ESIZE, TYPE are for the input, and LESIZE, LTYPE for the output.
964	*/
965	#define DO_2OP_L(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, FN) \
966	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void vn, void vm) \
967	{ \
968	LTYPE *d = vd; \
969	TYPE n = vn, m = vm; \
970	uint16_t mask = mve_element_mask(env); \
971	unsigned le; \
972	for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) { \
973	LTYPE r = FN((LTYPE)n[H##ESIZE(le * 2 + TOP)], \
974	m[H##ESIZE(le * 2 + TOP)]); \
975	mergemask(&d[H##LESIZE(le)], r, mask); \
976	} \
977	mve_advance_vpt(env); \
978	}
979
980	#define DO_2OP_SAT(OP, ESIZE, TYPE, FN) \
981	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void vn, void vm) \
982	{ \
983	TYPE d = vd, n = vn, *m = vm; \
984	uint16_t mask = mve_element_mask(env); \
985	unsigned e; \
986	bool qc = false; \
987	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
988	bool sat = false; \
989	TYPE r_ = FN(n[H##ESIZE(e)], m[H##ESIZE(e)], &sat); \
990	mergemask(&d[H##ESIZE(e)], r_, mask); \
991	qc \|= sat & mask & 1; \
992	} \
993	if (qc) { \
994	env->vfp.qc[0] = qc; \
995	} \
996	mve_advance_vpt(env); \
997	}
998
999	/* provide unsigned 2-op helpers for all sizes */
1000	#define DO_2OP_SAT_U(OP, FN) \
1001	DO_2OP_SAT(OP##b, 1, uint8_t, FN) \
1002	DO_2OP_SAT(OP##h, 2, uint16_t, FN) \
1003	DO_2OP_SAT(OP##w, 4, uint32_t, FN)
1004
1005	/* provide signed 2-op helpers for all sizes */
1006	#define DO_2OP_SAT_S(OP, FN) \
1007	DO_2OP_SAT(OP##b, 1, int8_t, FN) \
1008	DO_2OP_SAT(OP##h, 2, int16_t, FN) \
1009	DO_2OP_SAT(OP##w, 4, int32_t, FN)
1010
1011	#define DO_AND(N, M) ((N) & (M))
1012	#define DO_BIC(N, M) ((N) & ~(M))
1013	#define DO_ORR(N, M) ((N) \| (M))
1014	#define DO_ORN(N, M) ((N) \| ~(M))
1015	#define DO_EOR(N, M) ((N) ^ (M))
1016
1017	DO_2OP(vand, 8, uint64_t, DO_AND)
1018	DO_2OP(vbic, 8, uint64_t, DO_BIC)
1019	DO_2OP(vorr, 8, uint64_t, DO_ORR)
1020	DO_2OP(vorn, 8, uint64_t, DO_ORN)
1021	DO_2OP(veor, 8, uint64_t, DO_EOR)
1022
1023	#define DO_ADD(N, M) ((N) + (M))
1024	#define DO_SUB(N, M) ((N) - (M))
1025	#define DO_MUL(N, M) ((N) * (M))
1026
1027	DO_2OP_U(vadd, DO_ADD)
1028	DO_2OP_U(vsub, DO_SUB)
1029	DO_2OP_U(vmul, DO_MUL)
1030
1031	DO_2OP_L(vmullbsb, 0, 1, int8_t, 2, int16_t, DO_MUL)
1032	DO_2OP_L(vmullbsh, 0, 2, int16_t, 4, int32_t, DO_MUL)
1033	DO_2OP_L(vmullbsw, 0, 4, int32_t, 8, int64_t, DO_MUL)
1034	DO_2OP_L(vmullbub, 0, 1, uint8_t, 2, uint16_t, DO_MUL)
1035	DO_2OP_L(vmullbuh, 0, 2, uint16_t, 4, uint32_t, DO_MUL)
1036	DO_2OP_L(vmullbuw, 0, 4, uint32_t, 8, uint64_t, DO_MUL)
1037
1038	DO_2OP_L(vmulltsb, 1, 1, int8_t, 2, int16_t, DO_MUL)
1039	DO_2OP_L(vmulltsh, 1, 2, int16_t, 4, int32_t, DO_MUL)
1040	DO_2OP_L(vmulltsw, 1, 4, int32_t, 8, int64_t, DO_MUL)
1041	DO_2OP_L(vmulltub, 1, 1, uint8_t, 2, uint16_t, DO_MUL)
1042	DO_2OP_L(vmulltuh, 1, 2, uint16_t, 4, uint32_t, DO_MUL)
1043	DO_2OP_L(vmulltuw, 1, 4, uint32_t, 8, uint64_t, DO_MUL)
1044
1045	/*
1046	* Polynomial multiply. We can always do this generating 64 bits
1047	* of the result at a time, so we don't need to use DO_2OP_L.
1048	*/
1049	DO_2OP(vmullpbh, 8, uint64_t, clmul_8x4_even)
1050	DO_2OP(vmullpth, 8, uint64_t, clmul_8x4_odd)
1051	DO_2OP(vmullpbw, 8, uint64_t, clmul_16x2_even)
1052	DO_2OP(vmullptw, 8, uint64_t, clmul_16x2_odd)
1053
1054	/*
1055	* Because the computation type is at least twice as large as required,
1056	* these work for both signed and unsigned source types.
1057	*/
1058	static inline uint8_t do_mulh_b(int32_t n, int32_t m)
1059	{
1060	return (n * m) >> 8;
1061	}
1062
1063	static inline uint16_t do_mulh_h(int32_t n, int32_t m)
1064	{
1065	return (n * m) >> 16;
1066	}
1067
1068	static inline uint32_t do_mulh_w(int64_t n, int64_t m)
1069	{
1070	return (n * m) >> 32;
1071	}
1072
1073	static inline uint8_t do_rmulh_b(int32_t n, int32_t m)
1074	{
1075	return (n * m + (1U << 7)) >> 8;
1076	}
1077
1078	static inline uint16_t do_rmulh_h(int32_t n, int32_t m)
1079	{
1080	return (n * m + (1U << 15)) >> 16;
1081	}
1082
1083	static inline uint32_t do_rmulh_w(int64_t n, int64_t m)
1084	{
1085	return (n * m + (1U << 31)) >> 32;
1086	}
1087
1088	DO_2OP(vmulhsb, 1, int8_t, do_mulh_b)
1089	DO_2OP(vmulhsh, 2, int16_t, do_mulh_h)
1090	DO_2OP(vmulhsw, 4, int32_t, do_mulh_w)
1091	DO_2OP(vmulhub, 1, uint8_t, do_mulh_b)
1092	DO_2OP(vmulhuh, 2, uint16_t, do_mulh_h)
1093	DO_2OP(vmulhuw, 4, uint32_t, do_mulh_w)
1094
1095	DO_2OP(vrmulhsb, 1, int8_t, do_rmulh_b)
1096	DO_2OP(vrmulhsh, 2, int16_t, do_rmulh_h)
1097	DO_2OP(vrmulhsw, 4, int32_t, do_rmulh_w)
1098	DO_2OP(vrmulhub, 1, uint8_t, do_rmulh_b)
1099	DO_2OP(vrmulhuh, 2, uint16_t, do_rmulh_h)
1100	DO_2OP(vrmulhuw, 4, uint32_t, do_rmulh_w)
1101
1102	#define DO_MAX(N, M) ((N) >= (M) ? (N) : (M))
1103	#define DO_MIN(N, M) ((N) >= (M) ? (M) : (N))
1104
1105	DO_2OP_S(vmaxs, DO_MAX)
1106	DO_2OP_U(vmaxu, DO_MAX)
1107	DO_2OP_S(vmins, DO_MIN)
1108	DO_2OP_U(vminu, DO_MIN)
1109
1110	#define DO_ABD(N, M) ((N) >= (M) ? (N) - (M) : (M) - (N))
1111
1112	DO_2OP_S(vabds, DO_ABD)
1113	DO_2OP_U(vabdu, DO_ABD)
1114
1115	static inline uint32_t do_vhadd_u(uint32_t n, uint32_t m)
1116	{
1117	return ((uint64_t)n + m) >> 1;
1118	}
1119
1120	static inline int32_t do_vhadd_s(int32_t n, int32_t m)
1121	{
1122	return ((int64_t)n + m) >> 1;
1123	}
1124
1125	static inline uint32_t do_vhsub_u(uint32_t n, uint32_t m)
1126	{
1127	return ((uint64_t)n - m) >> 1;
1128	}
1129
1130	static inline int32_t do_vhsub_s(int32_t n, int32_t m)
1131	{
1132	return ((int64_t)n - m) >> 1;
1133	}
1134
1135	DO_2OP_S(vhadds, do_vhadd_s)
1136	DO_2OP_U(vhaddu, do_vhadd_u)
1137	DO_2OP_S(vhsubs, do_vhsub_s)
1138	DO_2OP_U(vhsubu, do_vhsub_u)
1139
1140	#define DO_VSHLS(N, M) do_sqrshl_bhs(N, (int8_t)(M), sizeof(N) * 8, false, NULL)
1141	#define DO_VSHLU(N, M) do_uqrshl_bhs(N, (int8_t)(M), sizeof(N) * 8, false, NULL)
1142	#define DO_VRSHLS(N, M) do_sqrshl_bhs(N, (int8_t)(M), sizeof(N) * 8, true, NULL)
1143	#define DO_VRSHLU(N, M) do_uqrshl_bhs(N, (int8_t)(M), sizeof(N) * 8, true, NULL)
1144
1145	DO_2OP_S(vshls, DO_VSHLS)
1146	DO_2OP_U(vshlu, DO_VSHLU)
1147	DO_2OP_S(vrshls, DO_VRSHLS)
1148	DO_2OP_U(vrshlu, DO_VRSHLU)
1149
1150	#define DO_RHADD_S(N, M) (((int64_t)(N) + (M) + 1) >> 1)
1151	#define DO_RHADD_U(N, M) (((uint64_t)(N) + (M) + 1) >> 1)
1152
1153	DO_2OP_S(vrhadds, DO_RHADD_S)
1154	DO_2OP_U(vrhaddu, DO_RHADD_U)
1155
1156	static void do_vadc(CPUARMState env, uint32_t d, uint32_t n, uint32_t m,
1157	uint32_t inv, uint32_t carry_in, bool update_flags)
1158	{
1159	uint16_t mask = mve_element_mask(env);
1160	unsigned e;
1161
1162	/* If any additions trigger, we will update flags. */
1163	if (mask & 0x1111) {
1164	update_flags = true;
1165	}
1166
1167	for (e = 0; e < 16 / 4; e++, mask >>= 4) {
1168	uint64_t r = carry_in;
1169	r += n[H4(e)];
1170	r += m[H4(e)] ^ inv;
1171	if (mask & 1) {
1172	carry_in = r >> 32;
1173	}
1174	mergemask(&d[H4(e)], r, mask);
1175	}
1176
1177	if (update_flags) {
1178	/* Store C, clear NZV. */
1179	env->vfp.fpsr &= ~FPSR_NZCV_MASK;
1180	env->vfp.fpsr \|= carry_in * FPSR_C;
1181	}
1182	mve_advance_vpt(env);
1183	}
1184
1185	void HELPER(mve_vadc)(CPUARMState env, void vd, void vn, void vm)
1186	{
1187	bool carry_in = env->vfp.fpsr & FPSR_C;
1188	do_vadc(env, vd, vn, vm, 0, carry_in, false);
1189	}
1190
1191	void HELPER(mve_vsbc)(CPUARMState env, void vd, void vn, void vm)
1192	{
1193	bool carry_in = env->vfp.fpsr & FPSR_C;
1194	do_vadc(env, vd, vn, vm, -1, carry_in, false);
1195	}
1196
1197
1198	void HELPER(mve_vadci)(CPUARMState env, void vd, void vn, void vm)
1199	{
1200	do_vadc(env, vd, vn, vm, 0, 0, true);
1201	}
1202
1203	void HELPER(mve_vsbci)(CPUARMState env, void vd, void vn, void vm)
1204	{
1205	do_vadc(env, vd, vn, vm, -1, 1, true);
1206	}
1207
1208	#define DO_VCADD(OP, ESIZE, TYPE, FN0, FN1) \
1209	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void vn, void vm) \
1210	{ \
1211	TYPE d = vd, n = vn, *m = vm; \
1212	uint16_t mask = mve_element_mask(env); \
1213	unsigned e; \
1214	TYPE r[16 / ESIZE]; \
1215	/* Calculate all results first to avoid overwriting inputs */ \
1216	for (e = 0; e < 16 / ESIZE; e++) { \
1217	if (!(e & 1)) { \
1218	r[e] = FN0(n[H##ESIZE(e)], m[H##ESIZE(e + 1)]); \
1219	} else { \
1220	r[e] = FN1(n[H##ESIZE(e)], m[H##ESIZE(e - 1)]); \
1221	} \
1222	} \
1223	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1224	mergemask(&d[H##ESIZE(e)], r[e], mask); \
1225	} \
1226	mve_advance_vpt(env); \
1227	}
1228
1229	#define DO_VCADD_ALL(OP, FN0, FN1) \
1230	DO_VCADD(OP##b, 1, int8_t, FN0, FN1) \
1231	DO_VCADD(OP##h, 2, int16_t, FN0, FN1) \
1232	DO_VCADD(OP##w, 4, int32_t, FN0, FN1)
1233
1234	DO_VCADD_ALL(vcadd90, DO_SUB, DO_ADD)
1235	DO_VCADD_ALL(vcadd270, DO_ADD, DO_SUB)
1236	DO_VCADD_ALL(vhcadd90, do_vhsub_s, do_vhadd_s)
1237	DO_VCADD_ALL(vhcadd270, do_vhadd_s, do_vhsub_s)
1238
1239	static inline int32_t do_sat_bhw(int64_t val, int64_t min, int64_t max, bool *s)
1240	{
1241	if (val > max) {
1242	*s = true;
1243	return max;
1244	} else if (val < min) {
1245	*s = true;
1246	return min;
1247	}
1248	return val;
1249	}
1250
1251	#define DO_SQADD_B(n, m, s) do_sat_bhw((int64_t)n + m, INT8_MIN, INT8_MAX, s)
1252	#define DO_SQADD_H(n, m, s) do_sat_bhw((int64_t)n + m, INT16_MIN, INT16_MAX, s)
1253	#define DO_SQADD_W(n, m, s) do_sat_bhw((int64_t)n + m, INT32_MIN, INT32_MAX, s)
1254
1255	#define DO_UQADD_B(n, m, s) do_sat_bhw((int64_t)n + m, 0, UINT8_MAX, s)
1256	#define DO_UQADD_H(n, m, s) do_sat_bhw((int64_t)n + m, 0, UINT16_MAX, s)
1257	#define DO_UQADD_W(n, m, s) do_sat_bhw((int64_t)n + m, 0, UINT32_MAX, s)
1258
1259	#define DO_SQSUB_B(n, m, s) do_sat_bhw((int64_t)n - m, INT8_MIN, INT8_MAX, s)
1260	#define DO_SQSUB_H(n, m, s) do_sat_bhw((int64_t)n - m, INT16_MIN, INT16_MAX, s)
1261	#define DO_SQSUB_W(n, m, s) do_sat_bhw((int64_t)n - m, INT32_MIN, INT32_MAX, s)
1262
1263	#define DO_UQSUB_B(n, m, s) do_sat_bhw((int64_t)n - m, 0, UINT8_MAX, s)
1264	#define DO_UQSUB_H(n, m, s) do_sat_bhw((int64_t)n - m, 0, UINT16_MAX, s)
1265	#define DO_UQSUB_W(n, m, s) do_sat_bhw((int64_t)n - m, 0, UINT32_MAX, s)
1266
1267	/*
1268	* For QDMULH and QRDMULH we simplify "double and shift by esize" into
1269	* "shift by esize-1", adjusting the QRDMULH rounding constant to match.
1270	*/
1271	#define DO_QDMULH_B(n, m, s) do_sat_bhw(((int64_t)n * m) >> 7, \
1272	INT8_MIN, INT8_MAX, s)
1273	#define DO_QDMULH_H(n, m, s) do_sat_bhw(((int64_t)n * m) >> 15, \
1274	INT16_MIN, INT16_MAX, s)
1275	#define DO_QDMULH_W(n, m, s) do_sat_bhw(((int64_t)n * m) >> 31, \
1276	INT32_MIN, INT32_MAX, s)
1277
1278	#define DO_QRDMULH_B(n, m, s) do_sat_bhw(((int64_t)n * m + (1 << 6)) >> 7, \
1279	INT8_MIN, INT8_MAX, s)
1280	#define DO_QRDMULH_H(n, m, s) do_sat_bhw(((int64_t)n * m + (1 << 14)) >> 15, \
1281	INT16_MIN, INT16_MAX, s)
1282	#define DO_QRDMULH_W(n, m, s) do_sat_bhw(((int64_t)n * m + (1 << 30)) >> 31, \
1283	INT32_MIN, INT32_MAX, s)
1284
1285	DO_2OP_SAT(vqdmulhb, 1, int8_t, DO_QDMULH_B)
1286	DO_2OP_SAT(vqdmulhh, 2, int16_t, DO_QDMULH_H)
1287	DO_2OP_SAT(vqdmulhw, 4, int32_t, DO_QDMULH_W)
1288
1289	DO_2OP_SAT(vqrdmulhb, 1, int8_t, DO_QRDMULH_B)
1290	DO_2OP_SAT(vqrdmulhh, 2, int16_t, DO_QRDMULH_H)
1291	DO_2OP_SAT(vqrdmulhw, 4, int32_t, DO_QRDMULH_W)
1292
1293	DO_2OP_SAT(vqaddub, 1, uint8_t, DO_UQADD_B)
1294	DO_2OP_SAT(vqadduh, 2, uint16_t, DO_UQADD_H)
1295	DO_2OP_SAT(vqadduw, 4, uint32_t, DO_UQADD_W)
1296	DO_2OP_SAT(vqaddsb, 1, int8_t, DO_SQADD_B)
1297	DO_2OP_SAT(vqaddsh, 2, int16_t, DO_SQADD_H)
1298	DO_2OP_SAT(vqaddsw, 4, int32_t, DO_SQADD_W)
1299
1300	DO_2OP_SAT(vqsubub, 1, uint8_t, DO_UQSUB_B)
1301	DO_2OP_SAT(vqsubuh, 2, uint16_t, DO_UQSUB_H)
1302	DO_2OP_SAT(vqsubuw, 4, uint32_t, DO_UQSUB_W)
1303	DO_2OP_SAT(vqsubsb, 1, int8_t, DO_SQSUB_B)
1304	DO_2OP_SAT(vqsubsh, 2, int16_t, DO_SQSUB_H)
1305	DO_2OP_SAT(vqsubsw, 4, int32_t, DO_SQSUB_W)
1306
1307	/*
1308	* This wrapper fixes up the impedance mismatch between do_sqrshl_bhs()
1309	* and friends wanting a uint32_t* sat and our needing a bool*.
1310	*/
1311	#define WRAP_QRSHL_HELPER(FN, N, M, ROUND, satp) \
1312	({ \
1313	uint32_t su32 = 0; \
1314	typeof(N) qrshl_ret = FN(N, (int8_t)(M), sizeof(N) * 8, ROUND, &su32); \
1315	if (su32) { \
1316	*satp = true; \
1317	} \
1318	qrshl_ret; \
1319	})
1320
1321	#define DO_SQSHL_OP(N, M, satp) \
1322	WRAP_QRSHL_HELPER(do_sqrshl_bhs, N, M, false, satp)
1323	#define DO_UQSHL_OP(N, M, satp) \
1324	WRAP_QRSHL_HELPER(do_uqrshl_bhs, N, M, false, satp)
1325	#define DO_SQRSHL_OP(N, M, satp) \
1326	WRAP_QRSHL_HELPER(do_sqrshl_bhs, N, M, true, satp)
1327	#define DO_UQRSHL_OP(N, M, satp) \
1328	WRAP_QRSHL_HELPER(do_uqrshl_bhs, N, M, true, satp)
1329	#define DO_SUQSHL_OP(N, M, satp) \
1330	WRAP_QRSHL_HELPER(do_suqrshl_bhs, N, M, false, satp)
1331
1332	DO_2OP_SAT_S(vqshls, DO_SQSHL_OP)
1333	DO_2OP_SAT_U(vqshlu, DO_UQSHL_OP)
1334	DO_2OP_SAT_S(vqrshls, DO_SQRSHL_OP)
1335	DO_2OP_SAT_U(vqrshlu, DO_UQRSHL_OP)
1336
1337	/*
1338	* Multiply add dual returning high half
1339	* The 'FN' here takes four inputs A, B, C, D, a 0/1 indicator of
1340	* whether to add the rounding constant, and the pointer to the
1341	* saturation flag, and should do "(A * B + C * D) * 2 + rounding constant",
1342	* saturate to twice the input size and return the high half; or
1343	* (A * B - C * D) etc for VQDMLSDH.
1344	*/
1345	#define DO_VQDMLADH_OP(OP, ESIZE, TYPE, XCHG, ROUND, FN) \
1346	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vn, \
1347	void *vm) \
1348	{ \
1349	TYPE d = vd, n = vn, *m = vm; \
1350	uint16_t mask = mve_element_mask(env); \
1351	unsigned e; \
1352	bool qc = false; \
1353	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1354	bool sat = false; \
1355	if ((e & 1) == XCHG) { \
1356	TYPE vqdmladh_ret = FN(n[H##ESIZE(e)], \
1357	m[H##ESIZE(e - XCHG)], \
1358	n[H##ESIZE(e + (1 - 2 * XCHG))], \
1359	m[H##ESIZE(e + (1 - XCHG))], \
1360	ROUND, &sat); \
1361	mergemask(&d[H##ESIZE(e)], vqdmladh_ret, mask); \
1362	qc \|= sat & mask & 1; \
1363	} \
1364	} \
1365	if (qc) { \
1366	env->vfp.qc[0] = qc; \
1367	} \
1368	mve_advance_vpt(env); \
1369	}
1370
1371	static int8_t do_vqdmladh_b(int8_t a, int8_t b, int8_t c, int8_t d,
1372	int round, bool *sat)
1373	{
1374	int64_t r = ((int64_t)a * b + (int64_t)c * d) * 2 + (round << 7);
1375	return do_sat_bhw(r, INT16_MIN, INT16_MAX, sat) >> 8;
1376	}
1377
1378	static int16_t do_vqdmladh_h(int16_t a, int16_t b, int16_t c, int16_t d,
1379	int round, bool *sat)
1380	{
1381	int64_t r = ((int64_t)a * b + (int64_t)c * d) * 2 + (round << 15);
1382	return do_sat_bhw(r, INT32_MIN, INT32_MAX, sat) >> 16;
1383	}
1384
1385	static int32_t do_vqdmladh_w(int32_t a, int32_t b, int32_t c, int32_t d,
1386	int round, bool *sat)
1387	{
1388	int64_t m1 = (int64_t)a * b;
1389	int64_t m2 = (int64_t)c * d;
1390	int64_t r;
1391	/*
1392	* Architecturally we should do the entire add, double, round
1393	* and then check for saturation. We do three saturating adds,
1394	* but we need to be careful about the order. If the first
1395	* m1 + m2 saturates then it's impossible for the *2+rc to
1396	* bring it back into the non-saturated range. However, if
1397	* m1 + m2 is negative then it's possible that doing the doubling
1398	* would take the intermediate result below INT64_MAX and the
1399	* addition of the rounding constant then brings it back in range.
1400	* So we add half the rounding constant before doubling rather
1401	* than adding the rounding constant after the doubling.
1402	*/
1403	if (sadd64_overflow(m1, m2, &r) \|\|
1404	sadd64_overflow(r, (round << 30), &r) \|\|
1405	sadd64_overflow(r, r, &r)) {
1406	*sat = true;
1407	return r < 0 ? INT32_MAX : INT32_MIN;
1408	}
1409	return r >> 32;
1410	}
1411
1412	static int8_t do_vqdmlsdh_b(int8_t a, int8_t b, int8_t c, int8_t d,
1413	int round, bool *sat)
1414	{
1415	int64_t r = ((int64_t)a * b - (int64_t)c * d) * 2 + (round << 7);
1416	return do_sat_bhw(r, INT16_MIN, INT16_MAX, sat) >> 8;
1417	}
1418
1419	static int16_t do_vqdmlsdh_h(int16_t a, int16_t b, int16_t c, int16_t d,
1420	int round, bool *sat)
1421	{
1422	int64_t r = ((int64_t)a * b - (int64_t)c * d) * 2 + (round << 15);
1423	return do_sat_bhw(r, INT32_MIN, INT32_MAX, sat) >> 16;
1424	}
1425
1426	static int32_t do_vqdmlsdh_w(int32_t a, int32_t b, int32_t c, int32_t d,
1427	int round, bool *sat)
1428	{
1429	int64_t m1 = (int64_t)a * b;
1430	int64_t m2 = (int64_t)c * d;
1431	int64_t r;
1432	/* The same ordering issue as in do_vqdmladh_w applies here too */
1433	if (ssub64_overflow(m1, m2, &r) \|\|
1434	sadd64_overflow(r, (round << 30), &r) \|\|
1435	sadd64_overflow(r, r, &r)) {
1436	*sat = true;
1437	return r < 0 ? INT32_MAX : INT32_MIN;
1438	}
1439	return r >> 32;
1440	}
1441
1442	DO_VQDMLADH_OP(vqdmladhb, 1, int8_t, 0, 0, do_vqdmladh_b)
1443	DO_VQDMLADH_OP(vqdmladhh, 2, int16_t, 0, 0, do_vqdmladh_h)
1444	DO_VQDMLADH_OP(vqdmladhw, 4, int32_t, 0, 0, do_vqdmladh_w)
1445	DO_VQDMLADH_OP(vqdmladhxb, 1, int8_t, 1, 0, do_vqdmladh_b)
1446	DO_VQDMLADH_OP(vqdmladhxh, 2, int16_t, 1, 0, do_vqdmladh_h)
1447	DO_VQDMLADH_OP(vqdmladhxw, 4, int32_t, 1, 0, do_vqdmladh_w)
1448
1449	DO_VQDMLADH_OP(vqrdmladhb, 1, int8_t, 0, 1, do_vqdmladh_b)
1450	DO_VQDMLADH_OP(vqrdmladhh, 2, int16_t, 0, 1, do_vqdmladh_h)
1451	DO_VQDMLADH_OP(vqrdmladhw, 4, int32_t, 0, 1, do_vqdmladh_w)
1452	DO_VQDMLADH_OP(vqrdmladhxb, 1, int8_t, 1, 1, do_vqdmladh_b)
1453	DO_VQDMLADH_OP(vqrdmladhxh, 2, int16_t, 1, 1, do_vqdmladh_h)
1454	DO_VQDMLADH_OP(vqrdmladhxw, 4, int32_t, 1, 1, do_vqdmladh_w)
1455
1456	DO_VQDMLADH_OP(vqdmlsdhb, 1, int8_t, 0, 0, do_vqdmlsdh_b)
1457	DO_VQDMLADH_OP(vqdmlsdhh, 2, int16_t, 0, 0, do_vqdmlsdh_h)
1458	DO_VQDMLADH_OP(vqdmlsdhw, 4, int32_t, 0, 0, do_vqdmlsdh_w)
1459	DO_VQDMLADH_OP(vqdmlsdhxb, 1, int8_t, 1, 0, do_vqdmlsdh_b)
1460	DO_VQDMLADH_OP(vqdmlsdhxh, 2, int16_t, 1, 0, do_vqdmlsdh_h)
1461	DO_VQDMLADH_OP(vqdmlsdhxw, 4, int32_t, 1, 0, do_vqdmlsdh_w)
1462
1463	DO_VQDMLADH_OP(vqrdmlsdhb, 1, int8_t, 0, 1, do_vqdmlsdh_b)
1464	DO_VQDMLADH_OP(vqrdmlsdhh, 2, int16_t, 0, 1, do_vqdmlsdh_h)
1465	DO_VQDMLADH_OP(vqrdmlsdhw, 4, int32_t, 0, 1, do_vqdmlsdh_w)
1466	DO_VQDMLADH_OP(vqrdmlsdhxb, 1, int8_t, 1, 1, do_vqdmlsdh_b)
1467	DO_VQDMLADH_OP(vqrdmlsdhxh, 2, int16_t, 1, 1, do_vqdmlsdh_h)
1468	DO_VQDMLADH_OP(vqrdmlsdhxw, 4, int32_t, 1, 1, do_vqdmlsdh_w)
1469
1470	#define DO_2OP_SCALAR(OP, ESIZE, TYPE, FN) \
1471	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vn, \
1472	uint32_t rm) \
1473	{ \
1474	TYPE d = vd, n = vn; \
1475	TYPE m = rm; \
1476	uint16_t mask = mve_element_mask(env); \
1477	unsigned e; \
1478	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1479	mergemask(&d[H##ESIZE(e)], FN(n[H##ESIZE(e)], m), mask); \
1480	} \
1481	mve_advance_vpt(env); \
1482	}
1483
1484	#define DO_2OP_SAT_SCALAR(OP, ESIZE, TYPE, FN) \
1485	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vn, \
1486	uint32_t rm) \
1487	{ \
1488	TYPE d = vd, n = vn; \
1489	TYPE m = rm; \
1490	uint16_t mask = mve_element_mask(env); \
1491	unsigned e; \
1492	bool qc = false; \
1493	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1494	bool sat = false; \
1495	mergemask(&d[H##ESIZE(e)], FN(n[H##ESIZE(e)], m, &sat), \
1496	mask); \
1497	qc \|= sat & mask & 1; \
1498	} \
1499	if (qc) { \
1500	env->vfp.qc[0] = qc; \
1501	} \
1502	mve_advance_vpt(env); \
1503	}
1504
1505	/* "accumulating" version where FN takes d as well as n and m */
1506	#define DO_2OP_ACC_SCALAR(OP, ESIZE, TYPE, FN) \
1507	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vn, \
1508	uint32_t rm) \
1509	{ \
1510	TYPE d = vd, n = vn; \
1511	TYPE m = rm; \
1512	uint16_t mask = mve_element_mask(env); \
1513	unsigned e; \
1514	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1515	mergemask(&d[H##ESIZE(e)], \
1516	FN(d[H##ESIZE(e)], n[H##ESIZE(e)], m), mask); \
1517	} \
1518	mve_advance_vpt(env); \
1519	}
1520
1521	#define DO_2OP_SAT_ACC_SCALAR(OP, ESIZE, TYPE, FN) \
1522	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vn, \
1523	uint32_t rm) \
1524	{ \
1525	TYPE d = vd, n = vn; \
1526	TYPE m = rm; \
1527	uint16_t mask = mve_element_mask(env); \
1528	unsigned e; \
1529	bool qc = false; \
1530	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1531	bool sat = false; \
1532	mergemask(&d[H##ESIZE(e)], \
1533	FN(d[H##ESIZE(e)], n[H##ESIZE(e)], m, &sat), \
1534	mask); \
1535	qc \|= sat & mask & 1; \
1536	} \
1537	if (qc) { \
1538	env->vfp.qc[0] = qc; \
1539	} \
1540	mve_advance_vpt(env); \
1541	}
1542
1543	/* provide unsigned 2-op scalar helpers for all sizes */
1544	#define DO_2OP_SCALAR_U(OP, FN) \
1545	DO_2OP_SCALAR(OP##b, 1, uint8_t, FN) \
1546	DO_2OP_SCALAR(OP##h, 2, uint16_t, FN) \
1547	DO_2OP_SCALAR(OP##w, 4, uint32_t, FN)
1548	#define DO_2OP_SCALAR_S(OP, FN) \
1549	DO_2OP_SCALAR(OP##b, 1, int8_t, FN) \
1550	DO_2OP_SCALAR(OP##h, 2, int16_t, FN) \
1551	DO_2OP_SCALAR(OP##w, 4, int32_t, FN)
1552
1553	#define DO_2OP_ACC_SCALAR_U(OP, FN) \
1554	DO_2OP_ACC_SCALAR(OP##b, 1, uint8_t, FN) \
1555	DO_2OP_ACC_SCALAR(OP##h, 2, uint16_t, FN) \
1556	DO_2OP_ACC_SCALAR(OP##w, 4, uint32_t, FN)
1557
1558	DO_2OP_SCALAR_U(vadd_scalar, DO_ADD)
1559	DO_2OP_SCALAR_U(vsub_scalar, DO_SUB)
1560	DO_2OP_SCALAR_U(vmul_scalar, DO_MUL)
1561	DO_2OP_SCALAR_S(vhadds_scalar, do_vhadd_s)
1562	DO_2OP_SCALAR_U(vhaddu_scalar, do_vhadd_u)
1563	DO_2OP_SCALAR_S(vhsubs_scalar, do_vhsub_s)
1564	DO_2OP_SCALAR_U(vhsubu_scalar, do_vhsub_u)
1565
1566	DO_2OP_SAT_SCALAR(vqaddu_scalarb, 1, uint8_t, DO_UQADD_B)
1567	DO_2OP_SAT_SCALAR(vqaddu_scalarh, 2, uint16_t, DO_UQADD_H)
1568	DO_2OP_SAT_SCALAR(vqaddu_scalarw, 4, uint32_t, DO_UQADD_W)
1569	DO_2OP_SAT_SCALAR(vqadds_scalarb, 1, int8_t, DO_SQADD_B)
1570	DO_2OP_SAT_SCALAR(vqadds_scalarh, 2, int16_t, DO_SQADD_H)
1571	DO_2OP_SAT_SCALAR(vqadds_scalarw, 4, int32_t, DO_SQADD_W)
1572
1573	DO_2OP_SAT_SCALAR(vqsubu_scalarb, 1, uint8_t, DO_UQSUB_B)
1574	DO_2OP_SAT_SCALAR(vqsubu_scalarh, 2, uint16_t, DO_UQSUB_H)
1575	DO_2OP_SAT_SCALAR(vqsubu_scalarw, 4, uint32_t, DO_UQSUB_W)
1576	DO_2OP_SAT_SCALAR(vqsubs_scalarb, 1, int8_t, DO_SQSUB_B)
1577	DO_2OP_SAT_SCALAR(vqsubs_scalarh, 2, int16_t, DO_SQSUB_H)
1578	DO_2OP_SAT_SCALAR(vqsubs_scalarw, 4, int32_t, DO_SQSUB_W)
1579
1580	DO_2OP_SAT_SCALAR(vqdmulh_scalarb, 1, int8_t, DO_QDMULH_B)
1581	DO_2OP_SAT_SCALAR(vqdmulh_scalarh, 2, int16_t, DO_QDMULH_H)
1582	DO_2OP_SAT_SCALAR(vqdmulh_scalarw, 4, int32_t, DO_QDMULH_W)
1583	DO_2OP_SAT_SCALAR(vqrdmulh_scalarb, 1, int8_t, DO_QRDMULH_B)
1584	DO_2OP_SAT_SCALAR(vqrdmulh_scalarh, 2, int16_t, DO_QRDMULH_H)
1585	DO_2OP_SAT_SCALAR(vqrdmulh_scalarw, 4, int32_t, DO_QRDMULH_W)
1586
1587	static int8_t do_vqdmlah_b(int8_t a, int8_t b, int8_t c, int round, bool *sat)
1588	{
1589	int64_t r = (int64_t)a * b * 2 + ((int64_t)c << 8) + (round << 7);
1590	return do_sat_bhw(r, INT16_MIN, INT16_MAX, sat) >> 8;
1591	}
1592
1593	static int16_t do_vqdmlah_h(int16_t a, int16_t b, int16_t c,
1594	int round, bool *sat)
1595	{
1596	int64_t r = (int64_t)a * b * 2 + ((int64_t)c << 16) + (round << 15);
1597	return do_sat_bhw(r, INT32_MIN, INT32_MAX, sat) >> 16;
1598	}
1599
1600	static int32_t do_vqdmlah_w(int32_t a, int32_t b, int32_t c,
1601	int round, bool *sat)
1602	{
1603	/*
1604	* Architecturally we should do the entire add, double, round
1605	* and then check for saturation. We do three saturating adds,
1606	* but we need to be careful about the order. If the first
1607	* m1 + m2 saturates then it's impossible for the *2+rc to
1608	* bring it back into the non-saturated range. However, if
1609	* m1 + m2 is negative then it's possible that doing the doubling
1610	* would take the intermediate result below INT64_MAX and the
1611	* addition of the rounding constant then brings it back in range.
1612	* So we add half the rounding constant and half the "c << esize"
1613	* before doubling rather than adding the rounding constant after
1614	* the doubling.
1615	*/
1616	int64_t m1 = (int64_t)a * b;
1617	int64_t m2 = (int64_t)c << 31;
1618	int64_t r;
1619	if (sadd64_overflow(m1, m2, &r) \|\|
1620	sadd64_overflow(r, (round << 30), &r) \|\|
1621	sadd64_overflow(r, r, &r)) {
1622	*sat = true;
1623	return r < 0 ? INT32_MAX : INT32_MIN;
1624	}
1625	return r >> 32;
1626	}
1627
1628	/*
1629	* The MLAH insns are vector scalar + vector;
1630	* the MLASH insns are vector vector + scalar
1631	*/
1632	#define DO_VQDMLAH_B(D, N, M, S) do_vqdmlah_b(N, M, D, 0, S)
1633	#define DO_VQDMLAH_H(D, N, M, S) do_vqdmlah_h(N, M, D, 0, S)
1634	#define DO_VQDMLAH_W(D, N, M, S) do_vqdmlah_w(N, M, D, 0, S)
1635	#define DO_VQRDMLAH_B(D, N, M, S) do_vqdmlah_b(N, M, D, 1, S)
1636	#define DO_VQRDMLAH_H(D, N, M, S) do_vqdmlah_h(N, M, D, 1, S)
1637	#define DO_VQRDMLAH_W(D, N, M, S) do_vqdmlah_w(N, M, D, 1, S)
1638
1639	#define DO_VQDMLASH_B(D, N, M, S) do_vqdmlah_b(N, D, M, 0, S)
1640	#define DO_VQDMLASH_H(D, N, M, S) do_vqdmlah_h(N, D, M, 0, S)
1641	#define DO_VQDMLASH_W(D, N, M, S) do_vqdmlah_w(N, D, M, 0, S)
1642	#define DO_VQRDMLASH_B(D, N, M, S) do_vqdmlah_b(N, D, M, 1, S)
1643	#define DO_VQRDMLASH_H(D, N, M, S) do_vqdmlah_h(N, D, M, 1, S)
1644	#define DO_VQRDMLASH_W(D, N, M, S) do_vqdmlah_w(N, D, M, 1, S)
1645
1646	DO_2OP_SAT_ACC_SCALAR(vqdmlahb, 1, int8_t, DO_VQDMLAH_B)
1647	DO_2OP_SAT_ACC_SCALAR(vqdmlahh, 2, int16_t, DO_VQDMLAH_H)
1648	DO_2OP_SAT_ACC_SCALAR(vqdmlahw, 4, int32_t, DO_VQDMLAH_W)
1649	DO_2OP_SAT_ACC_SCALAR(vqrdmlahb, 1, int8_t, DO_VQRDMLAH_B)
1650	DO_2OP_SAT_ACC_SCALAR(vqrdmlahh, 2, int16_t, DO_VQRDMLAH_H)
1651	DO_2OP_SAT_ACC_SCALAR(vqrdmlahw, 4, int32_t, DO_VQRDMLAH_W)
1652
1653	DO_2OP_SAT_ACC_SCALAR(vqdmlashb, 1, int8_t, DO_VQDMLASH_B)
1654	DO_2OP_SAT_ACC_SCALAR(vqdmlashh, 2, int16_t, DO_VQDMLASH_H)
1655	DO_2OP_SAT_ACC_SCALAR(vqdmlashw, 4, int32_t, DO_VQDMLASH_W)
1656	DO_2OP_SAT_ACC_SCALAR(vqrdmlashb, 1, int8_t, DO_VQRDMLASH_B)
1657	DO_2OP_SAT_ACC_SCALAR(vqrdmlashh, 2, int16_t, DO_VQRDMLASH_H)
1658	DO_2OP_SAT_ACC_SCALAR(vqrdmlashw, 4, int32_t, DO_VQRDMLASH_W)
1659
1660	/* Vector by scalar plus vector */
1661	#define DO_VMLA(D, N, M) ((N) * (M) + (D))
1662
1663	DO_2OP_ACC_SCALAR_U(vmla, DO_VMLA)
1664
1665	/* Vector by vector plus scalar */
1666	#define DO_VMLAS(D, N, M) ((N) * (D) + (M))
1667
1668	DO_2OP_ACC_SCALAR_U(vmlas, DO_VMLAS)
1669
1670	/*
1671	* Long saturating scalar ops. As with DO_2OP_L, TYPE and H are for the
1672	* input (smaller) type and LESIZE, LTYPE, LH for the output (long) type.
1673	* SATMASK specifies which bits of the predicate mask matter for determining
1674	* whether to propagate a saturation indication into FPSCR.QC -- for
1675	* the 16x16->32 case we must check only the bit corresponding to the T or B
1676	* half that we used, but for the 32x32->64 case we propagate if the mask
1677	* bit is set for either half.
1678	*/
1679	#define DO_2OP_SAT_SCALAR_L(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, FN, SATMASK) \
1680	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vn, \
1681	uint32_t rm) \
1682	{ \
1683	LTYPE *d = vd; \
1684	TYPE *n = vn; \
1685	TYPE m = rm; \
1686	uint16_t mask = mve_element_mask(env); \
1687	unsigned le; \
1688	bool qc = false; \
1689	for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) { \
1690	bool sat = false; \
1691	LTYPE r = FN((LTYPE)n[H##ESIZE(le * 2 + TOP)], m, &sat); \
1692	mergemask(&d[H##LESIZE(le)], r, mask); \
1693	qc \|= sat && (mask & SATMASK); \
1694	} \
1695	if (qc) { \
1696	env->vfp.qc[0] = qc; \
1697	} \
1698	mve_advance_vpt(env); \
1699	}
1700
1701	static inline int32_t do_qdmullh(int16_t n, int16_t m, bool *sat)
1702	{
1703	int64_t r = ((int64_t)n * m) * 2;
1704	return do_sat_bhw(r, INT32_MIN, INT32_MAX, sat);
1705	}
1706
1707	static inline int64_t do_qdmullw(int32_t n, int32_t m, bool *sat)
1708	{
1709	/* The multiply can't overflow, but the doubling might */
1710	int64_t r = (int64_t)n * m;
1711	if (r > INT64_MAX / 2) {
1712	*sat = true;
1713	return INT64_MAX;
1714	} else if (r < INT64_MIN / 2) {
1715	*sat = true;
1716	return INT64_MIN;
1717	} else {
1718	return r * 2;
1719	}
1720	}
1721
1722	#define SATMASK16B 1
1723	#define SATMASK16T (1 << 2)
1724	#define SATMASK32 ((1 << 4) \| 1)
1725
1726	DO_2OP_SAT_SCALAR_L(vqdmullb_scalarh, 0, 2, int16_t, 4, int32_t, \
1727	do_qdmullh, SATMASK16B)
1728	DO_2OP_SAT_SCALAR_L(vqdmullb_scalarw, 0, 4, int32_t, 8, int64_t, \
1729	do_qdmullw, SATMASK32)
1730	DO_2OP_SAT_SCALAR_L(vqdmullt_scalarh, 1, 2, int16_t, 4, int32_t, \
1731	do_qdmullh, SATMASK16T)
1732	DO_2OP_SAT_SCALAR_L(vqdmullt_scalarw, 1, 4, int32_t, 8, int64_t, \
1733	do_qdmullw, SATMASK32)
1734
1735	/*
1736	* Long saturating ops
1737	*/
1738	#define DO_2OP_SAT_L(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, FN, SATMASK) \
1739	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vn, \
1740	void *vm) \
1741	{ \
1742	LTYPE *d = vd; \
1743	TYPE n = vn, m = vm; \
1744	uint16_t mask = mve_element_mask(env); \
1745	unsigned le; \
1746	bool qc = false; \
1747	for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) { \
1748	bool sat = false; \
1749	LTYPE op1 = n[H##ESIZE(le * 2 + TOP)]; \
1750	LTYPE op2 = m[H##ESIZE(le * 2 + TOP)]; \
1751	mergemask(&d[H##LESIZE(le)], FN(op1, op2, &sat), mask); \
1752	qc \|= sat && (mask & SATMASK); \
1753	} \
1754	if (qc) { \
1755	env->vfp.qc[0] = qc; \
1756	} \
1757	mve_advance_vpt(env); \
1758	}
1759
1760	DO_2OP_SAT_L(vqdmullbh, 0, 2, int16_t, 4, int32_t, do_qdmullh, SATMASK16B)
1761	DO_2OP_SAT_L(vqdmullbw, 0, 4, int32_t, 8, int64_t, do_qdmullw, SATMASK32)
1762	DO_2OP_SAT_L(vqdmullth, 1, 2, int16_t, 4, int32_t, do_qdmullh, SATMASK16T)
1763	DO_2OP_SAT_L(vqdmulltw, 1, 4, int32_t, 8, int64_t, do_qdmullw, SATMASK32)
1764
1765	static inline uint32_t do_vbrsrb(uint32_t n, uint32_t m)
1766	{
1767	m &= 0xff;
1768	if (m == 0) {
1769	return 0;
1770	}
1771	n = revbit8(n);
1772	if (m < 8) {
1773	n >>= 8 - m;
1774	}
1775	return n;
1776	}
1777
1778	static inline uint32_t do_vbrsrh(uint32_t n, uint32_t m)
1779	{
1780	m &= 0xff;
1781	if (m == 0) {
1782	return 0;
1783	}
1784	n = revbit16(n);
1785	if (m < 16) {
1786	n >>= 16 - m;
1787	}
1788	return n;
1789	}
1790
1791	static inline uint32_t do_vbrsrw(uint32_t n, uint32_t m)
1792	{
1793	m &= 0xff;
1794	if (m == 0) {
1795	return 0;
1796	}
1797	n = revbit32(n);
1798	if (m < 32) {
1799	n >>= 32 - m;
1800	}
1801	return n;
1802	}
1803
1804	DO_2OP_SCALAR(vbrsrb, 1, uint8_t, do_vbrsrb)
1805	DO_2OP_SCALAR(vbrsrh, 2, uint16_t, do_vbrsrh)
1806	DO_2OP_SCALAR(vbrsrw, 4, uint32_t, do_vbrsrw)
1807
1808	/*
1809	* Multiply add long dual accumulate ops.
1810	*/
1811	#define DO_LDAV(OP, ESIZE, TYPE, XCHG, EVENACC, ODDACC) \
1812	uint64_t HELPER(glue(mve_, OP))(CPUARMState env, void vn, \
1813	void *vm, uint64_t a) \
1814	{ \
1815	uint16_t mask = mve_element_mask(env); \
1816	unsigned e; \
1817	TYPE n = vn, m = vm; \
1818	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1819	if (mask & 1) { \
1820	if (e & 1) { \
1821	a ODDACC \
1822	(int64_t)n[H##ESIZE(e - 1 * XCHG)] * m[H##ESIZE(e)]; \
1823	} else { \
1824	a EVENACC \
1825	(int64_t)n[H##ESIZE(e + 1 * XCHG)] * m[H##ESIZE(e)]; \
1826	} \
1827	} \
1828	} \
1829	mve_advance_vpt(env); \
1830	return a; \
1831	}
1832
1833	DO_LDAV(vmlaldavsh, 2, int16_t, false, +=, +=)
1834	DO_LDAV(vmlaldavxsh, 2, int16_t, true, +=, +=)
1835	DO_LDAV(vmlaldavsw, 4, int32_t, false, +=, +=)
1836	DO_LDAV(vmlaldavxsw, 4, int32_t, true, +=, +=)
1837
1838	DO_LDAV(vmlaldavuh, 2, uint16_t, false, +=, +=)
1839	DO_LDAV(vmlaldavuw, 4, uint32_t, false, +=, +=)
1840
1841	DO_LDAV(vmlsldavsh, 2, int16_t, false, +=, -=)
1842	DO_LDAV(vmlsldavxsh, 2, int16_t, true, +=, -=)
1843	DO_LDAV(vmlsldavsw, 4, int32_t, false, +=, -=)
1844	DO_LDAV(vmlsldavxsw, 4, int32_t, true, +=, -=)
1845
1846	/*
1847	* Multiply add dual accumulate ops
1848	*/
1849	#define DO_DAV(OP, ESIZE, TYPE, XCHG, EVENACC, ODDACC) \
1850	uint32_t HELPER(glue(mve_, OP))(CPUARMState env, void vn, \
1851	void *vm, uint32_t a) \
1852	{ \
1853	uint16_t mask = mve_element_mask(env); \
1854	unsigned e; \
1855	TYPE n = vn, m = vm; \
1856	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1857	if (mask & 1) { \
1858	if (e & 1) { \
1859	a ODDACC \
1860	n[H##ESIZE(e - 1 * XCHG)] * m[H##ESIZE(e)]; \
1861	} else { \
1862	a EVENACC \
1863	n[H##ESIZE(e + 1 * XCHG)] * m[H##ESIZE(e)]; \
1864	} \
1865	} \
1866	} \
1867	mve_advance_vpt(env); \
1868	return a; \
1869	}
1870
1871	#define DO_DAV_S(INSN, XCHG, EVENACC, ODDACC) \
1872	DO_DAV(INSN##b, 1, int8_t, XCHG, EVENACC, ODDACC) \
1873	DO_DAV(INSN##h, 2, int16_t, XCHG, EVENACC, ODDACC) \
1874	DO_DAV(INSN##w, 4, int32_t, XCHG, EVENACC, ODDACC)
1875
1876	#define DO_DAV_U(INSN, XCHG, EVENACC, ODDACC) \
1877	DO_DAV(INSN##b, 1, uint8_t, XCHG, EVENACC, ODDACC) \
1878	DO_DAV(INSN##h, 2, uint16_t, XCHG, EVENACC, ODDACC) \
1879	DO_DAV(INSN##w, 4, uint32_t, XCHG, EVENACC, ODDACC)
1880
1881	DO_DAV_S(vmladavs, false, +=, +=)
1882	DO_DAV_U(vmladavu, false, +=, +=)
1883	DO_DAV_S(vmlsdav, false, +=, -=)
1884	DO_DAV_S(vmladavsx, true, +=, +=)
1885	DO_DAV_S(vmlsdavx, true, +=, -=)
1886
1887	/*
1888	* Rounding multiply add long dual accumulate high. In the pseudocode
1889	* this is implemented with a 72-bit internal accumulator value of which
1890	* the top 64 bits are returned. We optimize this to avoid having to
1891	* use 128-bit arithmetic -- we can do this because the 74-bit accumulator
1892	* is squashed back into 64-bits after each beat.
1893	*/
1894	#define DO_LDAVH(OP, TYPE, LTYPE, XCHG, SUB) \
1895	uint64_t HELPER(glue(mve_, OP))(CPUARMState env, void vn, \
1896	void *vm, uint64_t a) \
1897	{ \
1898	uint16_t mask = mve_element_mask(env); \
1899	unsigned e; \
1900	TYPE n = vn, m = vm; \
1901	for (e = 0; e < 16 / 4; e++, mask >>= 4) { \
1902	if (mask & 1) { \
1903	LTYPE mul; \
1904	if (e & 1) { \
1905	mul = (LTYPE)n[H4(e - 1 * XCHG)] * m[H4(e)]; \
1906	if (SUB) { \
1907	mul = -mul; \
1908	} \
1909	} else { \
1910	mul = (LTYPE)n[H4(e + 1 * XCHG)] * m[H4(e)]; \
1911	} \
1912	mul = (mul >> 8) + ((mul >> 7) & 1); \
1913	a += mul; \
1914	} \
1915	} \
1916	mve_advance_vpt(env); \
1917	return a; \
1918	}
1919
1920	DO_LDAVH(vrmlaldavhsw, int32_t, int64_t, false, false)
1921	DO_LDAVH(vrmlaldavhxsw, int32_t, int64_t, true, false)
1922
1923	DO_LDAVH(vrmlaldavhuw, uint32_t, uint64_t, false, false)
1924
1925	DO_LDAVH(vrmlsldavhsw, int32_t, int64_t, false, true)
1926	DO_LDAVH(vrmlsldavhxsw, int32_t, int64_t, true, true)
1927
1928	/* Vector add across vector */
1929	#define DO_VADDV(OP, ESIZE, TYPE) \
1930	uint32_t HELPER(glue(mve_, OP))(CPUARMState env, void vm, \
1931	uint32_t ra) \
1932	{ \
1933	uint16_t mask = mve_element_mask(env); \
1934	unsigned e; \
1935	TYPE *m = vm; \
1936	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1937	if (mask & 1) { \
1938	ra += m[H##ESIZE(e)]; \
1939	} \
1940	} \
1941	mve_advance_vpt(env); \
1942	return ra; \
1943	} \
1944
1945	DO_VADDV(vaddvsb, 1, int8_t)
1946	DO_VADDV(vaddvsh, 2, int16_t)
1947	DO_VADDV(vaddvsw, 4, int32_t)
1948	DO_VADDV(vaddvub, 1, uint8_t)
1949	DO_VADDV(vaddvuh, 2, uint16_t)
1950	DO_VADDV(vaddvuw, 4, uint32_t)
1951
1952	/*
1953	* Vector max/min across vector. Unlike VADDV, we must
1954	* read ra as the element size, not its full width.
1955	* We work with int64_t internally for simplicity.
1956	*/
1957	#define DO_VMAXMINV(OP, ESIZE, TYPE, RATYPE, FN) \
1958	uint32_t HELPER(glue(mve_, OP))(CPUARMState env, void vm, \
1959	uint32_t ra_in) \
1960	{ \
1961	uint16_t mask = mve_element_mask(env); \
1962	unsigned e; \
1963	TYPE *m = vm; \
1964	int64_t ra = (RATYPE)ra_in; \
1965	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
1966	if (mask & 1) { \
1967	ra = FN(ra, m[H##ESIZE(e)]); \
1968	} \
1969	} \
1970	mve_advance_vpt(env); \
1971	return ra; \
1972	} \
1973
1974	#define DO_VMAXMINV_U(INSN, FN) \
1975	DO_VMAXMINV(INSN##b, 1, uint8_t, uint8_t, FN) \
1976	DO_VMAXMINV(INSN##h, 2, uint16_t, uint16_t, FN) \
1977	DO_VMAXMINV(INSN##w, 4, uint32_t, uint32_t, FN)
1978	#define DO_VMAXMINV_S(INSN, FN) \
1979	DO_VMAXMINV(INSN##b, 1, int8_t, int8_t, FN) \
1980	DO_VMAXMINV(INSN##h, 2, int16_t, int16_t, FN) \
1981	DO_VMAXMINV(INSN##w, 4, int32_t, int32_t, FN)
1982
1983	/*
1984	* Helpers for max and min of absolute values across vector:
1985	* note that we only take the absolute value of 'm', not 'n'
1986	*/
1987	static int64_t do_maxa(int64_t n, int64_t m)
1988	{
1989	if (m < 0) {
1990	m = -m;
1991	}
1992	return MAX(n, m);
1993	}
1994
1995	static int64_t do_mina(int64_t n, int64_t m)
1996	{
1997	if (m < 0) {
1998	m = -m;
1999	}
2000	return MIN(n, m);
2001	}
2002
2003	DO_VMAXMINV_S(vmaxvs, DO_MAX)
2004	DO_VMAXMINV_U(vmaxvu, DO_MAX)
2005	DO_VMAXMINV_S(vminvs, DO_MIN)
2006	DO_VMAXMINV_U(vminvu, DO_MIN)
2007	/*
2008	* VMAXAV, VMINAV treat the general purpose input as unsigned
2009	* and the vector elements as signed.
2010	*/
2011	DO_VMAXMINV(vmaxavb, 1, int8_t, uint8_t, do_maxa)
2012	DO_VMAXMINV(vmaxavh, 2, int16_t, uint16_t, do_maxa)
2013	DO_VMAXMINV(vmaxavw, 4, int32_t, uint32_t, do_maxa)
2014	DO_VMAXMINV(vminavb, 1, int8_t, uint8_t, do_mina)
2015	DO_VMAXMINV(vminavh, 2, int16_t, uint16_t, do_mina)
2016	DO_VMAXMINV(vminavw, 4, int32_t, uint32_t, do_mina)
2017
2018	#define DO_VABAV(OP, ESIZE, TYPE) \
2019	uint32_t HELPER(glue(mve_, OP))(CPUARMState env, void vn, \
2020	void *vm, uint32_t ra) \
2021	{ \
2022	uint16_t mask = mve_element_mask(env); \
2023	unsigned e; \
2024	TYPE m = vm, n = vn; \
2025	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2026	if (mask & 1) { \
2027	int64_t n0 = n[H##ESIZE(e)]; \
2028	int64_t m0 = m[H##ESIZE(e)]; \
2029	uint32_t r = n0 >= m0 ? (n0 - m0) : (m0 - n0); \
2030	ra += r; \
2031	} \
2032	} \
2033	mve_advance_vpt(env); \
2034	return ra; \
2035	}
2036
2037	DO_VABAV(vabavsb, 1, int8_t)
2038	DO_VABAV(vabavsh, 2, int16_t)
2039	DO_VABAV(vabavsw, 4, int32_t)
2040	DO_VABAV(vabavub, 1, uint8_t)
2041	DO_VABAV(vabavuh, 2, uint16_t)
2042	DO_VABAV(vabavuw, 4, uint32_t)
2043
2044	#define DO_VADDLV(OP, TYPE, LTYPE) \
2045	uint64_t HELPER(glue(mve_, OP))(CPUARMState env, void vm, \
2046	uint64_t ra) \
2047	{ \
2048	uint16_t mask = mve_element_mask(env); \
2049	unsigned e; \
2050	TYPE *m = vm; \
2051	for (e = 0; e < 16 / 4; e++, mask >>= 4) { \
2052	if (mask & 1) { \
2053	ra += (LTYPE)m[H4(e)]; \
2054	} \
2055	} \
2056	mve_advance_vpt(env); \
2057	return ra; \
2058	} \
2059
2060	DO_VADDLV(vaddlv_s, int32_t, int64_t)
2061	DO_VADDLV(vaddlv_u, uint32_t, uint64_t)
2062
2063	/* Shifts by immediate */
2064	#define DO_2SHIFT(OP, ESIZE, TYPE, FN) \
2065	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, \
2066	void *vm, uint32_t shift) \
2067	{ \
2068	TYPE d = vd, m = vm; \
2069	uint16_t mask = mve_element_mask(env); \
2070	unsigned e; \
2071	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2072	mergemask(&d[H##ESIZE(e)], \
2073	FN(m[H##ESIZE(e)], shift), mask); \
2074	} \
2075	mve_advance_vpt(env); \
2076	}
2077
2078	#define DO_2SHIFT_SAT(OP, ESIZE, TYPE, FN) \
2079	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, \
2080	void *vm, uint32_t shift) \
2081	{ \
2082	TYPE d = vd, m = vm; \
2083	uint16_t mask = mve_element_mask(env); \
2084	unsigned e; \
2085	bool qc = false; \
2086	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2087	bool sat = false; \
2088	mergemask(&d[H##ESIZE(e)], \
2089	FN(m[H##ESIZE(e)], shift, &sat), mask); \
2090	qc \|= sat & mask & 1; \
2091	} \
2092	if (qc) { \
2093	env->vfp.qc[0] = qc; \
2094	} \
2095	mve_advance_vpt(env); \
2096	}
2097
2098	/* provide unsigned 2-op shift helpers for all sizes */
2099	#define DO_2SHIFT_U(OP, FN) \
2100	DO_2SHIFT(OP##b, 1, uint8_t, FN) \
2101	DO_2SHIFT(OP##h, 2, uint16_t, FN) \
2102	DO_2SHIFT(OP##w, 4, uint32_t, FN)
2103	#define DO_2SHIFT_S(OP, FN) \
2104	DO_2SHIFT(OP##b, 1, int8_t, FN) \
2105	DO_2SHIFT(OP##h, 2, int16_t, FN) \
2106	DO_2SHIFT(OP##w, 4, int32_t, FN)
2107
2108	#define DO_2SHIFT_SAT_U(OP, FN) \
2109	DO_2SHIFT_SAT(OP##b, 1, uint8_t, FN) \
2110	DO_2SHIFT_SAT(OP##h, 2, uint16_t, FN) \
2111	DO_2SHIFT_SAT(OP##w, 4, uint32_t, FN)
2112	#define DO_2SHIFT_SAT_S(OP, FN) \
2113	DO_2SHIFT_SAT(OP##b, 1, int8_t, FN) \
2114	DO_2SHIFT_SAT(OP##h, 2, int16_t, FN) \
2115	DO_2SHIFT_SAT(OP##w, 4, int32_t, FN)
2116
2117	DO_2SHIFT_U(vshli_u, DO_VSHLU)
2118	DO_2SHIFT_S(vshli_s, DO_VSHLS)
2119	DO_2SHIFT_SAT_U(vqshli_u, DO_UQSHL_OP)
2120	DO_2SHIFT_SAT_S(vqshli_s, DO_SQSHL_OP)
2121	DO_2SHIFT_SAT_S(vqshlui_s, DO_SUQSHL_OP)
2122	DO_2SHIFT_U(vrshli_u, DO_VRSHLU)
2123	DO_2SHIFT_S(vrshli_s, DO_VRSHLS)
2124	DO_2SHIFT_SAT_U(vqrshli_u, DO_UQRSHL_OP)
2125	DO_2SHIFT_SAT_S(vqrshli_s, DO_SQRSHL_OP)
2126
2127	/* Shift-and-insert; we always work with 64 bits at a time */
2128	#define DO_2SHIFT_INSERT(OP, ESIZE, SHIFTFN, MASKFN) \
2129	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, \
2130	void *vm, uint32_t shift) \
2131	{ \
2132	uint64_t d = vd, m = vm; \
2133	uint16_t mask; \
2134	uint64_t shiftmask; \
2135	unsigned e; \
2136	if (shift == ESIZE * 8) { \
2137	/* \
2138	* Only VSRI can shift by <dt>; it should mean "don't \
2139	* update the destination". The generic logic can't handle \
2140	* this because it would try to shift by an out-of-range \
2141	* amount, so special case it here. \
2142	*/ \
2143	goto done; \
2144	} \
2145	assert(shift < ESIZE * 8); \
2146	mask = mve_element_mask(env); \
2147	/* ESIZE / 2 gives the MO_* value if ESIZE is in [1,2,4] */ \
2148	shiftmask = dup_const(ESIZE / 2, MASKFN(ESIZE * 8, shift)); \
2149	for (e = 0; e < 16 / 8; e++, mask >>= 8) { \
2150	uint64_t r = (SHIFTFN(m[H8(e)], shift) & shiftmask) \| \
2151	(d[H8(e)] & ~shiftmask); \
2152	mergemask(&d[H8(e)], r, mask); \
2153	} \
2154	done: \
2155	mve_advance_vpt(env); \
2156	}
2157
2158	#define DO_SHL(N, SHIFT) ((N) << (SHIFT))
2159	#define DO_SHR(N, SHIFT) ((N) >> (SHIFT))
2160	#define SHL_MASK(EBITS, SHIFT) MAKE_64BIT_MASK((SHIFT), (EBITS) - (SHIFT))
2161	#define SHR_MASK(EBITS, SHIFT) MAKE_64BIT_MASK(0, (EBITS) - (SHIFT))
2162
2163	DO_2SHIFT_INSERT(vsrib, 1, DO_SHR, SHR_MASK)
2164	DO_2SHIFT_INSERT(vsrih, 2, DO_SHR, SHR_MASK)
2165	DO_2SHIFT_INSERT(vsriw, 4, DO_SHR, SHR_MASK)
2166	DO_2SHIFT_INSERT(vslib, 1, DO_SHL, SHL_MASK)
2167	DO_2SHIFT_INSERT(vslih, 2, DO_SHL, SHL_MASK)
2168	DO_2SHIFT_INSERT(vsliw, 4, DO_SHL, SHL_MASK)
2169
2170	/*
2171	* Long shifts taking half-sized inputs from top or bottom of the input
2172	* vector and producing a double-width result. ESIZE, TYPE are for
2173	* the input, and LESIZE, LTYPE for the output.
2174	* Unlike the normal shift helpers, we do not handle negative shift counts,
2175	* because the long shift is strictly left-only.
2176	*/
2177	#define DO_VSHLL(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE) \
2178	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, \
2179	void *vm, uint32_t shift) \
2180	{ \
2181	LTYPE *d = vd; \
2182	TYPE *m = vm; \
2183	uint16_t mask = mve_element_mask(env); \
2184	unsigned le; \
2185	assert(shift <= 16); \
2186	for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) { \
2187	LTYPE r = (LTYPE)m[H##ESIZE(le * 2 + TOP)] << shift; \
2188	mergemask(&d[H##LESIZE(le)], r, mask); \
2189	} \
2190	mve_advance_vpt(env); \
2191	}
2192
2193	#define DO_VSHLL_ALL(OP, TOP) \
2194	DO_VSHLL(OP##sb, TOP, 1, int8_t, 2, int16_t) \
2195	DO_VSHLL(OP##ub, TOP, 1, uint8_t, 2, uint16_t) \
2196	DO_VSHLL(OP##sh, TOP, 2, int16_t, 4, int32_t) \
2197	DO_VSHLL(OP##uh, TOP, 2, uint16_t, 4, uint32_t) \
2198
2199	DO_VSHLL_ALL(vshllb, false)
2200	DO_VSHLL_ALL(vshllt, true)
2201
2202	/*
2203	* Narrowing right shifts, taking a double sized input, shifting it
2204	* and putting the result in either the top or bottom half of the output.
2205	* ESIZE, TYPE are the output, and LESIZE, LTYPE the input.
2206	*/
2207	#define DO_VSHRN(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, FN) \
2208	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, \
2209	void *vm, uint32_t shift) \
2210	{ \
2211	LTYPE *m = vm; \
2212	TYPE *d = vd; \
2213	uint16_t mask = mve_element_mask(env); \
2214	unsigned le; \
2215	mask >>= ESIZE * TOP; \
2216	for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) { \
2217	TYPE r = FN(m[H##LESIZE(le)], shift); \
2218	mergemask(&d[H##ESIZE(le * 2 + TOP)], r, mask); \
2219	} \
2220	mve_advance_vpt(env); \
2221	}
2222
2223	#define DO_VSHRN_ALL(OP, FN) \
2224	DO_VSHRN(OP##bb, false, 1, uint8_t, 2, uint16_t, FN) \
2225	DO_VSHRN(OP##bh, false, 2, uint16_t, 4, uint32_t, FN) \
2226	DO_VSHRN(OP##tb, true, 1, uint8_t, 2, uint16_t, FN) \
2227	DO_VSHRN(OP##th, true, 2, uint16_t, 4, uint32_t, FN)
2228
2229	DO_VSHRN_ALL(vshrn, DO_SHR)
2230	DO_VSHRN_ALL(vrshrn, do_urshr)
2231
2232	static inline int32_t do_sat_bhs(int64_t val, int64_t min, int64_t max,
2233	bool *satp)
2234	{
2235	if (val > max) {
2236	*satp = true;
2237	return max;
2238	} else if (val < min) {
2239	*satp = true;
2240	return min;
2241	} else {
2242	return val;
2243	}
2244	}
2245
2246	/* Saturating narrowing right shifts */
2247	#define DO_VSHRN_SAT(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, FN) \
2248	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, \
2249	void *vm, uint32_t shift) \
2250	{ \
2251	LTYPE *m = vm; \
2252	TYPE *d = vd; \
2253	uint16_t mask = mve_element_mask(env); \
2254	bool qc = false; \
2255	unsigned le; \
2256	mask >>= ESIZE * TOP; \
2257	for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) { \
2258	bool sat = false; \
2259	TYPE r = FN(m[H##LESIZE(le)], shift, &sat); \
2260	mergemask(&d[H##ESIZE(le * 2 + TOP)], r, mask); \
2261	qc \|= sat & mask & 1; \
2262	} \
2263	if (qc) { \
2264	env->vfp.qc[0] = qc; \
2265	} \
2266	mve_advance_vpt(env); \
2267	}
2268
2269	#define DO_VSHRN_SAT_UB(BOP, TOP, FN) \
2270	DO_VSHRN_SAT(BOP, false, 1, uint8_t, 2, uint16_t, FN) \
2271	DO_VSHRN_SAT(TOP, true, 1, uint8_t, 2, uint16_t, FN)
2272
2273	#define DO_VSHRN_SAT_UH(BOP, TOP, FN) \
2274	DO_VSHRN_SAT(BOP, false, 2, uint16_t, 4, uint32_t, FN) \
2275	DO_VSHRN_SAT(TOP, true, 2, uint16_t, 4, uint32_t, FN)
2276
2277	#define DO_VSHRN_SAT_SB(BOP, TOP, FN) \
2278	DO_VSHRN_SAT(BOP, false, 1, int8_t, 2, int16_t, FN) \
2279	DO_VSHRN_SAT(TOP, true, 1, int8_t, 2, int16_t, FN)
2280
2281	#define DO_VSHRN_SAT_SH(BOP, TOP, FN) \
2282	DO_VSHRN_SAT(BOP, false, 2, int16_t, 4, int32_t, FN) \
2283	DO_VSHRN_SAT(TOP, true, 2, int16_t, 4, int32_t, FN)
2284
2285	#define DO_SHRN_SB(N, M, SATP) \
2286	do_sat_bhs((int64_t)(N) >> (M), INT8_MIN, INT8_MAX, SATP)
2287	#define DO_SHRN_UB(N, M, SATP) \
2288	do_sat_bhs((uint64_t)(N) >> (M), 0, UINT8_MAX, SATP)
2289	#define DO_SHRUN_B(N, M, SATP) \
2290	do_sat_bhs((int64_t)(N) >> (M), 0, UINT8_MAX, SATP)
2291
2292	#define DO_SHRN_SH(N, M, SATP) \
2293	do_sat_bhs((int64_t)(N) >> (M), INT16_MIN, INT16_MAX, SATP)
2294	#define DO_SHRN_UH(N, M, SATP) \
2295	do_sat_bhs((uint64_t)(N) >> (M), 0, UINT16_MAX, SATP)
2296	#define DO_SHRUN_H(N, M, SATP) \
2297	do_sat_bhs((int64_t)(N) >> (M), 0, UINT16_MAX, SATP)
2298
2299	#define DO_RSHRN_SB(N, M, SATP) \
2300	do_sat_bhs(do_srshr(N, M), INT8_MIN, INT8_MAX, SATP)
2301	#define DO_RSHRN_UB(N, M, SATP) \
2302	do_sat_bhs(do_urshr(N, M), 0, UINT8_MAX, SATP)
2303	#define DO_RSHRUN_B(N, M, SATP) \
2304	do_sat_bhs(do_srshr(N, M), 0, UINT8_MAX, SATP)
2305
2306	#define DO_RSHRN_SH(N, M, SATP) \
2307	do_sat_bhs(do_srshr(N, M), INT16_MIN, INT16_MAX, SATP)
2308	#define DO_RSHRN_UH(N, M, SATP) \
2309	do_sat_bhs(do_urshr(N, M), 0, UINT16_MAX, SATP)
2310	#define DO_RSHRUN_H(N, M, SATP) \
2311	do_sat_bhs(do_srshr(N, M), 0, UINT16_MAX, SATP)
2312
2313	DO_VSHRN_SAT_SB(vqshrnb_sb, vqshrnt_sb, DO_SHRN_SB)
2314	DO_VSHRN_SAT_SH(vqshrnb_sh, vqshrnt_sh, DO_SHRN_SH)
2315	DO_VSHRN_SAT_UB(vqshrnb_ub, vqshrnt_ub, DO_SHRN_UB)
2316	DO_VSHRN_SAT_UH(vqshrnb_uh, vqshrnt_uh, DO_SHRN_UH)
2317	DO_VSHRN_SAT_SB(vqshrunbb, vqshruntb, DO_SHRUN_B)
2318	DO_VSHRN_SAT_SH(vqshrunbh, vqshrunth, DO_SHRUN_H)
2319
2320	DO_VSHRN_SAT_SB(vqrshrnb_sb, vqrshrnt_sb, DO_RSHRN_SB)
2321	DO_VSHRN_SAT_SH(vqrshrnb_sh, vqrshrnt_sh, DO_RSHRN_SH)
2322	DO_VSHRN_SAT_UB(vqrshrnb_ub, vqrshrnt_ub, DO_RSHRN_UB)
2323	DO_VSHRN_SAT_UH(vqrshrnb_uh, vqrshrnt_uh, DO_RSHRN_UH)
2324	DO_VSHRN_SAT_SB(vqrshrunbb, vqrshruntb, DO_RSHRUN_B)
2325	DO_VSHRN_SAT_SH(vqrshrunbh, vqrshrunth, DO_RSHRUN_H)
2326
2327	#define DO_VMOVN(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE) \
2328	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm) \
2329	{ \
2330	LTYPE *m = vm; \
2331	TYPE *d = vd; \
2332	uint16_t mask = mve_element_mask(env); \
2333	unsigned le; \
2334	mask >>= ESIZE * TOP; \
2335	for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) { \
2336	mergemask(&d[H##ESIZE(le * 2 + TOP)], \
2337	m[H##LESIZE(le)], mask); \
2338	} \
2339	mve_advance_vpt(env); \
2340	}
2341
2342	DO_VMOVN(vmovnbb, false, 1, uint8_t, 2, uint16_t)
2343	DO_VMOVN(vmovnbh, false, 2, uint16_t, 4, uint32_t)
2344	DO_VMOVN(vmovntb, true, 1, uint8_t, 2, uint16_t)
2345	DO_VMOVN(vmovnth, true, 2, uint16_t, 4, uint32_t)
2346
2347	#define DO_VMOVN_SAT(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, FN) \
2348	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm) \
2349	{ \
2350	LTYPE *m = vm; \
2351	TYPE *d = vd; \
2352	uint16_t mask = mve_element_mask(env); \
2353	bool qc = false; \
2354	unsigned le; \
2355	mask >>= ESIZE * TOP; \
2356	for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) { \
2357	bool sat = false; \
2358	TYPE r = FN(m[H##LESIZE(le)], &sat); \
2359	mergemask(&d[H##ESIZE(le * 2 + TOP)], r, mask); \
2360	qc \|= sat & mask & 1; \
2361	} \
2362	if (qc) { \
2363	env->vfp.qc[0] = qc; \
2364	} \
2365	mve_advance_vpt(env); \
2366	}
2367
2368	#define DO_VMOVN_SAT_UB(BOP, TOP, FN) \
2369	DO_VMOVN_SAT(BOP, false, 1, uint8_t, 2, uint16_t, FN) \
2370	DO_VMOVN_SAT(TOP, true, 1, uint8_t, 2, uint16_t, FN)
2371
2372	#define DO_VMOVN_SAT_UH(BOP, TOP, FN) \
2373	DO_VMOVN_SAT(BOP, false, 2, uint16_t, 4, uint32_t, FN) \
2374	DO_VMOVN_SAT(TOP, true, 2, uint16_t, 4, uint32_t, FN)
2375
2376	#define DO_VMOVN_SAT_SB(BOP, TOP, FN) \
2377	DO_VMOVN_SAT(BOP, false, 1, int8_t, 2, int16_t, FN) \
2378	DO_VMOVN_SAT(TOP, true, 1, int8_t, 2, int16_t, FN)
2379
2380	#define DO_VMOVN_SAT_SH(BOP, TOP, FN) \
2381	DO_VMOVN_SAT(BOP, false, 2, int16_t, 4, int32_t, FN) \
2382	DO_VMOVN_SAT(TOP, true, 2, int16_t, 4, int32_t, FN)
2383
2384	#define DO_VQMOVN_SB(N, SATP) \
2385	do_sat_bhs((int64_t)(N), INT8_MIN, INT8_MAX, SATP)
2386	#define DO_VQMOVN_UB(N, SATP) \
2387	do_sat_bhs((uint64_t)(N), 0, UINT8_MAX, SATP)
2388	#define DO_VQMOVUN_B(N, SATP) \
2389	do_sat_bhs((int64_t)(N), 0, UINT8_MAX, SATP)
2390
2391	#define DO_VQMOVN_SH(N, SATP) \
2392	do_sat_bhs((int64_t)(N), INT16_MIN, INT16_MAX, SATP)
2393	#define DO_VQMOVN_UH(N, SATP) \
2394	do_sat_bhs((uint64_t)(N), 0, UINT16_MAX, SATP)
2395	#define DO_VQMOVUN_H(N, SATP) \
2396	do_sat_bhs((int64_t)(N), 0, UINT16_MAX, SATP)
2397
2398	DO_VMOVN_SAT_SB(vqmovnbsb, vqmovntsb, DO_VQMOVN_SB)
2399	DO_VMOVN_SAT_SH(vqmovnbsh, vqmovntsh, DO_VQMOVN_SH)
2400	DO_VMOVN_SAT_UB(vqmovnbub, vqmovntub, DO_VQMOVN_UB)
2401	DO_VMOVN_SAT_UH(vqmovnbuh, vqmovntuh, DO_VQMOVN_UH)
2402	DO_VMOVN_SAT_SB(vqmovunbb, vqmovuntb, DO_VQMOVUN_B)
2403	DO_VMOVN_SAT_SH(vqmovunbh, vqmovunth, DO_VQMOVUN_H)
2404
2405	uint32_t HELPER(mve_vshlc)(CPUARMState env, void vd, uint32_t rdm,
2406	uint32_t shift)
2407	{
2408	uint32_t *d = vd;
2409	uint16_t mask = mve_element_mask(env);
2410	unsigned e;
2411	uint32_t r;
2412
2413	/*
2414	* For each 32-bit element, we shift it left, bringing in the
2415	* low 'shift' bits of rdm at the bottom. Bits shifted out at
2416	* the top become the new rdm, if the predicate mask permits.
2417	* The final rdm value is returned to update the register.
2418	* shift == 0 here means "shift by 32 bits".
2419	*/
2420	if (shift == 0) {
2421	for (e = 0; e < 16 / 4; e++, mask >>= 4) {
2422	r = rdm;
2423	if (mask & 1) {
2424	rdm = d[H4(e)];
2425	}
2426	mergemask(&d[H4(e)], r, mask);
2427	}
2428	} else {
2429	uint32_t shiftmask = MAKE_64BIT_MASK(0, shift);
2430
2431	for (e = 0; e < 16 / 4; e++, mask >>= 4) {
2432	r = (d[H4(e)] << shift) \| (rdm & shiftmask);
2433	if (mask & 1) {
2434	rdm = d[H4(e)] >> (32 - shift);
2435	}
2436	mergemask(&d[H4(e)], r, mask);
2437	}
2438	}
2439	mve_advance_vpt(env);
2440	return rdm;
2441	}
2442
2443	uint64_t HELPER(mve_sshrl)(CPUARMState *env, uint64_t n, uint32_t shift)
2444	{
2445	return do_sqrshl_d(n, -(int8_t)shift, false, NULL);
2446	}
2447
2448	uint64_t HELPER(mve_ushll)(CPUARMState *env, uint64_t n, uint32_t shift)
2449	{
2450	return do_uqrshl_d(n, (int8_t)shift, false, NULL);
2451	}
2452
2453	uint64_t HELPER(mve_sqshll)(CPUARMState *env, uint64_t n, uint32_t shift)
2454	{
2455	return do_sqrshl_d(n, (int8_t)shift, false, &env->QF);
2456	}
2457
2458	uint64_t HELPER(mve_uqshll)(CPUARMState *env, uint64_t n, uint32_t shift)
2459	{
2460	return do_uqrshl_d(n, (int8_t)shift, false, &env->QF);
2461	}
2462
2463	uint64_t HELPER(mve_sqrshrl)(CPUARMState *env, uint64_t n, uint32_t shift)
2464	{
2465	return do_sqrshl_d(n, -(int8_t)shift, true, &env->QF);
2466	}
2467
2468	uint64_t HELPER(mve_uqrshll)(CPUARMState *env, uint64_t n, uint32_t shift)
2469	{
2470	return do_uqrshl_d(n, (int8_t)shift, true, &env->QF);
2471	}
2472
2473	/* Operate on 64-bit values, but saturate at 48 bits */
2474	static inline int64_t do_sqrshl48_d(int64_t src, int64_t shift,
2475	bool round, uint32_t *sat)
2476	{
2477	int64_t val, extval;
2478
2479	if (shift <= -48) {
2480	/* Rounding the sign bit always produces 0. */
2481	if (round) {
2482	return 0;
2483	}
2484	return src >> 63;
2485	} else if (shift < 0) {
2486	if (round) {
2487	src >>= -shift - 1;
2488	val = (src >> 1) + (src & 1);
2489	} else {
2490	val = src >> -shift;
2491	}
2492	extval = sextract64(val, 0, 48);
2493	if (!sat \|\| val == extval) {
2494	return extval;
2495	}
2496	} else if (shift < 48) {
2497	extval = sextract64(src << shift, 0, 48);
2498	if (!sat \|\| src == (extval >> shift)) {
2499	return extval;
2500	}
2501	} else if (!sat \|\| src == 0) {
2502	return 0;
2503	}
2504
2505	*sat = 1;
2506	return src >= 0 ? MAKE_64BIT_MASK(0, 47) : MAKE_64BIT_MASK(47, 17);
2507	}
2508
2509	/* Operate on 64-bit values, but saturate at 48 bits */
2510	static inline uint64_t do_uqrshl48_d(uint64_t src, int64_t shift,
2511	bool round, uint32_t *sat)
2512	{
2513	uint64_t val, extval;
2514
2515	if (shift <= -(48 + round)) {
2516	return 0;
2517	} else if (shift < 0) {
2518	if (round) {
2519	val = src >> (-shift - 1);
2520	val = (val >> 1) + (val & 1);
2521	} else {
2522	val = src >> -shift;
2523	}
2524	extval = extract64(val, 0, 48);
2525	if (!sat \|\| val == extval) {
2526	return extval;
2527	}
2528	} else if (shift < 48) {
2529	extval = extract64(src << shift, 0, 48);
2530	if (!sat \|\| src == (extval >> shift)) {
2531	return extval;
2532	}
2533	} else if (!sat \|\| src == 0) {
2534	return 0;
2535	}
2536
2537	*sat = 1;
2538	return MAKE_64BIT_MASK(0, 48);
2539	}
2540
2541	uint64_t HELPER(mve_sqrshrl48)(CPUARMState *env, uint64_t n, uint32_t shift)
2542	{
2543	return do_sqrshl48_d(n, -(int8_t)shift, true, &env->QF);
2544	}
2545
2546	uint64_t HELPER(mve_uqrshll48)(CPUARMState *env, uint64_t n, uint32_t shift)
2547	{
2548	return do_uqrshl48_d(n, (int8_t)shift, true, &env->QF);
2549	}
2550
2551	uint32_t HELPER(mve_uqshl)(CPUARMState *env, uint32_t n, uint32_t shift)
2552	{
2553	return do_uqrshl_bhs(n, (int8_t)shift, 32, false, &env->QF);
2554	}
2555
2556	uint32_t HELPER(mve_sqshl)(CPUARMState *env, uint32_t n, uint32_t shift)
2557	{
2558	return do_sqrshl_bhs(n, (int8_t)shift, 32, false, &env->QF);
2559	}
2560
2561	uint32_t HELPER(mve_uqrshl)(CPUARMState *env, uint32_t n, uint32_t shift)
2562	{
2563	return do_uqrshl_bhs(n, (int8_t)shift, 32, true, &env->QF);
2564	}
2565
2566	uint32_t HELPER(mve_sqrshr)(CPUARMState *env, uint32_t n, uint32_t shift)
2567	{
2568	return do_sqrshl_bhs(n, -(int8_t)shift, 32, true, &env->QF);
2569	}
2570
2571	#define DO_VIDUP(OP, ESIZE, TYPE, FN) \
2572	uint32_t HELPER(mve_##OP)(CPUARMState env, void vd, \
2573	uint32_t offset, uint32_t imm) \
2574	{ \
2575	TYPE *d = vd; \
2576	uint16_t mask = mve_element_mask(env); \
2577	unsigned e; \
2578	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2579	mergemask(&d[H##ESIZE(e)], offset, mask); \
2580	offset = FN(offset, imm); \
2581	} \
2582	mve_advance_vpt(env); \
2583	return offset; \
2584	}
2585
2586	#define DO_VIWDUP(OP, ESIZE, TYPE, FN) \
2587	uint32_t HELPER(mve_##OP)(CPUARMState env, void vd, \
2588	uint32_t offset, uint32_t wrap, \
2589	uint32_t imm) \
2590	{ \
2591	TYPE *d = vd; \
2592	uint16_t mask = mve_element_mask(env); \
2593	unsigned e; \
2594	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2595	mergemask(&d[H##ESIZE(e)], offset, mask); \
2596	offset = FN(offset, wrap, imm); \
2597	} \
2598	mve_advance_vpt(env); \
2599	return offset; \
2600	}
2601
2602	#define DO_VIDUP_ALL(OP, FN) \
2603	DO_VIDUP(OP##b, 1, int8_t, FN) \
2604	DO_VIDUP(OP##h, 2, int16_t, FN) \
2605	DO_VIDUP(OP##w, 4, int32_t, FN)
2606
2607	#define DO_VIWDUP_ALL(OP, FN) \
2608	DO_VIWDUP(OP##b, 1, int8_t, FN) \
2609	DO_VIWDUP(OP##h, 2, int16_t, FN) \
2610	DO_VIWDUP(OP##w, 4, int32_t, FN)
2611
2612	static uint32_t do_add_wrap(uint32_t offset, uint32_t wrap, uint32_t imm)
2613	{
2614	offset += imm;
2615	if (offset == wrap) {
2616	offset = 0;
2617	}
2618	return offset;
2619	}
2620
2621	static uint32_t do_sub_wrap(uint32_t offset, uint32_t wrap, uint32_t imm)
2622	{
2623	if (offset == 0) {
2624	offset = wrap;
2625	}
2626	offset -= imm;
2627	return offset;
2628	}
2629
2630	DO_VIDUP_ALL(vidup, DO_ADD)
2631	DO_VIWDUP_ALL(viwdup, do_add_wrap)
2632	DO_VIWDUP_ALL(vdwdup, do_sub_wrap)
2633
2634	/*
2635	* Vector comparison.
2636	* P0 bits for non-executed beats (where eci_mask is 0) are unchanged.
2637	* P0 bits for predicated lanes in executed beats (where mask is 0) are 0.
2638	* P0 bits otherwise are updated with the results of the comparisons.
2639	* We must also keep unchanged the MASK fields at the top of v7m.vpr.
2640	*/
2641	#define DO_VCMP(OP, ESIZE, TYPE, FN) \
2642	void HELPER(glue(mve_, OP))(CPUARMState env, void vn, void *vm) \
2643	{ \
2644	TYPE n = vn, m = vm; \
2645	uint16_t mask = mve_element_mask(env); \
2646	uint16_t eci_mask = mve_eci_mask(env); \
2647	uint16_t beatpred = 0; \
2648	uint16_t emask = MAKE_64BIT_MASK(0, ESIZE); \
2649	unsigned e; \
2650	for (e = 0; e < 16 / ESIZE; e++) { \
2651	bool r = FN(n[H##ESIZE(e)], m[H##ESIZE(e)]); \
2652	/* Comparison sets 0/1 bits for each byte in the element */ \
2653	beatpred \|= r * emask; \
2654	emask <<= ESIZE; \
2655	} \
2656	beatpred &= mask; \
2657	env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) \| \
2658	(beatpred & eci_mask); \
2659	mve_advance_vpt(env); \
2660	}
2661
2662	#define DO_VCMP_SCALAR(OP, ESIZE, TYPE, FN) \
2663	void HELPER(glue(mve_, OP))(CPUARMState env, void vn, \
2664	uint32_t rm) \
2665	{ \
2666	TYPE *n = vn; \
2667	uint16_t mask = mve_element_mask(env); \
2668	uint16_t eci_mask = mve_eci_mask(env); \
2669	uint16_t beatpred = 0; \
2670	uint16_t emask = MAKE_64BIT_MASK(0, ESIZE); \
2671	unsigned e; \
2672	for (e = 0; e < 16 / ESIZE; e++) { \
2673	bool r = FN(n[H##ESIZE(e)], (TYPE)rm); \
2674	/* Comparison sets 0/1 bits for each byte in the element */ \
2675	beatpred \|= r * emask; \
2676	emask <<= ESIZE; \
2677	} \
2678	beatpred &= mask; \
2679	env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) \| \
2680	(beatpred & eci_mask); \
2681	mve_advance_vpt(env); \
2682	}
2683
2684	#define DO_VCMP_S(OP, FN) \
2685	DO_VCMP(OP##b, 1, int8_t, FN) \
2686	DO_VCMP(OP##h, 2, int16_t, FN) \
2687	DO_VCMP(OP##w, 4, int32_t, FN) \
2688	DO_VCMP_SCALAR(OP##_scalarb, 1, int8_t, FN) \
2689	DO_VCMP_SCALAR(OP##_scalarh, 2, int16_t, FN) \
2690	DO_VCMP_SCALAR(OP##_scalarw, 4, int32_t, FN)
2691
2692	#define DO_VCMP_U(OP, FN) \
2693	DO_VCMP(OP##b, 1, uint8_t, FN) \
2694	DO_VCMP(OP##h, 2, uint16_t, FN) \
2695	DO_VCMP(OP##w, 4, uint32_t, FN) \
2696	DO_VCMP_SCALAR(OP##_scalarb, 1, uint8_t, FN) \
2697	DO_VCMP_SCALAR(OP##_scalarh, 2, uint16_t, FN) \
2698	DO_VCMP_SCALAR(OP##_scalarw, 4, uint32_t, FN)
2699
2700	#define DO_EQ(N, M) ((N) == (M))
2701	#define DO_NE(N, M) ((N) != (M))
2702	#define DO_EQ(N, M) ((N) == (M))
2703	#define DO_EQ(N, M) ((N) == (M))
2704	#define DO_GE(N, M) ((N) >= (M))
2705	#define DO_LT(N, M) ((N) < (M))
2706	#define DO_GT(N, M) ((N) > (M))
2707	#define DO_LE(N, M) ((N) <= (M))
2708
2709	DO_VCMP_U(vcmpeq, DO_EQ)
2710	DO_VCMP_U(vcmpne, DO_NE)
2711	DO_VCMP_U(vcmpcs, DO_GE)
2712	DO_VCMP_U(vcmphi, DO_GT)
2713	DO_VCMP_S(vcmpge, DO_GE)
2714	DO_VCMP_S(vcmplt, DO_LT)
2715	DO_VCMP_S(vcmpgt, DO_GT)
2716	DO_VCMP_S(vcmple, DO_LE)
2717
2718	void HELPER(mve_vpsel)(CPUARMState env, void vd, void vn, void vm)
2719	{
2720	/*
2721	* Qd[n] = VPR.P0[n] ? Qn[n] : Qm[n]
2722	* but note that whether bytes are written to Qd is still subject
2723	* to (all forms of) predication in the usual way.
2724	*/
2725	uint64_t d = vd, n = vn, *m = vm;
2726	uint16_t mask = mve_element_mask(env);
2727	uint16_t p0 = FIELD_EX32(env->v7m.vpr, V7M_VPR, P0);
2728	unsigned e;
2729	for (e = 0; e < 16 / 8; e++, mask >>= 8, p0 >>= 8) {
2730	uint64_t r = m[H8(e)];
2731	mergemask(&r, n[H8(e)], p0);
2732	mergemask(&d[H8(e)], r, mask);
2733	}
2734	mve_advance_vpt(env);
2735	}
2736
2737	void HELPER(mve_vpnot)(CPUARMState *env)
2738	{
2739	/*
2740	* P0 bits for unexecuted beats (where eci_mask is 0) are unchanged.
2741	* P0 bits for predicated lanes in executed bits (where mask is 0) are 0.
2742	* P0 bits otherwise are inverted.
2743	* (This is the same logic as VCMP.)
2744	* This insn is itself subject to predication and to beat-wise execution,
2745	* and after it executes VPT state advances in the usual way.
2746	*/
2747	uint16_t mask = mve_element_mask(env);
2748	uint16_t eci_mask = mve_eci_mask(env);
2749	uint16_t beatpred = ~env->v7m.vpr & mask;
2750	env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) \| (beatpred & eci_mask);
2751	mve_advance_vpt(env);
2752	}
2753
2754	/*
2755	* VCTP: P0 unexecuted bits unchanged, predicated bits zeroed,
2756	* otherwise set according to value of Rn. The calculation of
2757	* newmask here works in the same way as the calculation of the
2758	* ltpmask in mve_element_mask(), but we have pre-calculated
2759	* the masklen in the generated code.
2760	*/
2761	void HELPER(mve_vctp)(CPUARMState *env, uint32_t masklen)
2762	{
2763	uint16_t mask = mve_element_mask(env);
2764	uint16_t eci_mask = mve_eci_mask(env);
2765	uint16_t newmask;
2766
2767	assert(masklen <= 16);
2768	newmask = masklen ? MAKE_64BIT_MASK(0, masklen) : 0;
2769	newmask &= mask;
2770	env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) \| (newmask & eci_mask);
2771	mve_advance_vpt(env);
2772	}
2773
2774	#define DO_1OP_SAT(OP, ESIZE, TYPE, FN) \
2775	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm) \
2776	{ \
2777	TYPE d = vd, m = vm; \
2778	uint16_t mask = mve_element_mask(env); \
2779	unsigned e; \
2780	bool qc = false; \
2781	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2782	bool sat = false; \
2783	mergemask(&d[H##ESIZE(e)], FN(m[H##ESIZE(e)], &sat), mask); \
2784	qc \|= sat & mask & 1; \
2785	} \
2786	if (qc) { \
2787	env->vfp.qc[0] = qc; \
2788	} \
2789	mve_advance_vpt(env); \
2790	}
2791
2792	#define DO_VQABS_B(N, SATP) \
2793	do_sat_bhs(DO_ABS((int64_t)N), INT8_MIN, INT8_MAX, SATP)
2794	#define DO_VQABS_H(N, SATP) \
2795	do_sat_bhs(DO_ABS((int64_t)N), INT16_MIN, INT16_MAX, SATP)
2796	#define DO_VQABS_W(N, SATP) \
2797	do_sat_bhs(DO_ABS((int64_t)N), INT32_MIN, INT32_MAX, SATP)
2798
2799	#define DO_VQNEG_B(N, SATP) do_sat_bhs(-(int64_t)N, INT8_MIN, INT8_MAX, SATP)
2800	#define DO_VQNEG_H(N, SATP) do_sat_bhs(-(int64_t)N, INT16_MIN, INT16_MAX, SATP)
2801	#define DO_VQNEG_W(N, SATP) do_sat_bhs(-(int64_t)N, INT32_MIN, INT32_MAX, SATP)
2802
2803	DO_1OP_SAT(vqabsb, 1, int8_t, DO_VQABS_B)
2804	DO_1OP_SAT(vqabsh, 2, int16_t, DO_VQABS_H)
2805	DO_1OP_SAT(vqabsw, 4, int32_t, DO_VQABS_W)
2806
2807	DO_1OP_SAT(vqnegb, 1, int8_t, DO_VQNEG_B)
2808	DO_1OP_SAT(vqnegh, 2, int16_t, DO_VQNEG_H)
2809	DO_1OP_SAT(vqnegw, 4, int32_t, DO_VQNEG_W)
2810
2811	/*
2812	* VMAXA, VMINA: vd is unsigned; vm is signed, and we take its
2813	* absolute value; we then do an unsigned comparison.
2814	*/
2815	#define DO_VMAXMINA(OP, ESIZE, STYPE, UTYPE, FN) \
2816	void HELPER(mve_##OP)(CPUARMState env, void vd, void *vm) \
2817	{ \
2818	UTYPE *d = vd; \
2819	STYPE *m = vm; \
2820	uint16_t mask = mve_element_mask(env); \
2821	unsigned e; \
2822	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2823	UTYPE r = DO_ABS(m[H##ESIZE(e)]); \
2824	r = FN(d[H##ESIZE(e)], r); \
2825	mergemask(&d[H##ESIZE(e)], r, mask); \
2826	} \
2827	mve_advance_vpt(env); \
2828	}
2829
2830	DO_VMAXMINA(vmaxab, 1, int8_t, uint8_t, DO_MAX)
2831	DO_VMAXMINA(vmaxah, 2, int16_t, uint16_t, DO_MAX)
2832	DO_VMAXMINA(vmaxaw, 4, int32_t, uint32_t, DO_MAX)
2833	DO_VMAXMINA(vminab, 1, int8_t, uint8_t, DO_MIN)
2834	DO_VMAXMINA(vminah, 2, int16_t, uint16_t, DO_MIN)
2835	DO_VMAXMINA(vminaw, 4, int32_t, uint32_t, DO_MIN)
2836
2837	/*
2838	* 2-operand floating point. Note that if an element is partially
2839	* predicated we must do the FP operation to update the non-predicated
2840	* bytes, but we must be careful to avoid updating the FP exception
2841	* state unless byte 0 of the element was unpredicated.
2842	*/
2843	#define DO_2OP_FP(OP, ESIZE, TYPE, FN) \
2844	void HELPER(glue(mve_, OP))(CPUARMState *env, \
2845	void vd, void vn, void *vm) \
2846	{ \
2847	TYPE d = vd, n = vn, *m = vm; \
2848	TYPE r; \
2849	uint16_t mask = mve_element_mask(env); \
2850	unsigned e; \
2851	float_status *fpst; \
2852	float_status scratch_fpst; \
2853	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2854	if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) { \
2855	continue; \
2856	} \
2857	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
2858	if (!(mask & 1)) { \
2859	/* We need the result but without updating flags */ \
2860	scratch_fpst = *fpst; \
2861	fpst = &scratch_fpst; \
2862	} \
2863	r = FN(n[H##ESIZE(e)], m[H##ESIZE(e)], fpst); \
2864	mergemask(&d[H##ESIZE(e)], r, mask); \
2865	} \
2866	mve_advance_vpt(env); \
2867	}
2868
2869	#define DO_2OP_FP_ALL(OP, FN) \
2870	DO_2OP_FP(OP##h, 2, float16, float16_##FN) \
2871	DO_2OP_FP(OP##s, 4, float32, float32_##FN)
2872
2873	DO_2OP_FP_ALL(vfadd, add)
2874	DO_2OP_FP_ALL(vfsub, sub)
2875	DO_2OP_FP_ALL(vfmul, mul)
2876
2877	static inline float16 float16_abd(float16 a, float16 b, float_status *s)
2878	{
2879	return float16_abs(float16_sub(a, b, s));
2880	}
2881
2882	static inline float32 float32_abd(float32 a, float32 b, float_status *s)
2883	{
2884	return float32_abs(float32_sub(a, b, s));
2885	}
2886
2887	DO_2OP_FP_ALL(vfabd, abd)
2888	DO_2OP_FP_ALL(vmaxnm, maxnum)
2889	DO_2OP_FP_ALL(vminnm, minnum)
2890
2891	static inline float16 float16_maxnuma(float16 a, float16 b, float_status *s)
2892	{
2893	return float16_maxnum(float16_abs(a), float16_abs(b), s);
2894	}
2895
2896	static inline float32 float32_maxnuma(float32 a, float32 b, float_status *s)
2897	{
2898	return float32_maxnum(float32_abs(a), float32_abs(b), s);
2899	}
2900
2901	static inline float16 float16_minnuma(float16 a, float16 b, float_status *s)
2902	{
2903	return float16_minnum(float16_abs(a), float16_abs(b), s);
2904	}
2905
2906	static inline float32 float32_minnuma(float32 a, float32 b, float_status *s)
2907	{
2908	return float32_minnum(float32_abs(a), float32_abs(b), s);
2909	}
2910
2911	DO_2OP_FP_ALL(vmaxnma, maxnuma)
2912	DO_2OP_FP_ALL(vminnma, minnuma)
2913
2914	#define DO_VCADD_FP(OP, ESIZE, TYPE, FN0, FN1) \
2915	void HELPER(glue(mve_, OP))(CPUARMState *env, \
2916	void vd, void vn, void *vm) \
2917	{ \
2918	TYPE d = vd, n = vn, *m = vm; \
2919	TYPE r[16 / ESIZE]; \
2920	uint16_t tm, mask = mve_element_mask(env); \
2921	unsigned e; \
2922	float_status *fpst; \
2923	float_status scratch_fpst; \
2924	/* Calculate all results first to avoid overwriting inputs */ \
2925	for (e = 0, tm = mask; e < 16 / ESIZE; e++, tm >>= ESIZE) { \
2926	if ((tm & MAKE_64BIT_MASK(0, ESIZE)) == 0) { \
2927	r[e] = 0; \
2928	continue; \
2929	} \
2930	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
2931	if (!(tm & 1)) { \
2932	/* We need the result but without updating flags */ \
2933	scratch_fpst = *fpst; \
2934	fpst = &scratch_fpst; \
2935	} \
2936	if (!(e & 1)) { \
2937	r[e] = FN0(n[H##ESIZE(e)], m[H##ESIZE(e + 1)], fpst); \
2938	} else { \
2939	r[e] = FN1(n[H##ESIZE(e)], m[H##ESIZE(e - 1)], fpst); \
2940	} \
2941	} \
2942	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2943	mergemask(&d[H##ESIZE(e)], r[e], mask); \
2944	} \
2945	mve_advance_vpt(env); \
2946	}
2947
2948	DO_VCADD_FP(vfcadd90h, 2, float16, float16_sub, float16_add)
2949	DO_VCADD_FP(vfcadd90s, 4, float32, float32_sub, float32_add)
2950	DO_VCADD_FP(vfcadd270h, 2, float16, float16_add, float16_sub)
2951	DO_VCADD_FP(vfcadd270s, 4, float32, float32_add, float32_sub)
2952
2953	#define DO_VFMA(OP, ESIZE, TYPE, CHS) \
2954	void HELPER(glue(mve_, OP))(CPUARMState *env, \
2955	void vd, void vn, void *vm) \
2956	{ \
2957	TYPE d = vd, n = vn, *m = vm; \
2958	TYPE r; \
2959	uint16_t mask = mve_element_mask(env); \
2960	unsigned e; \
2961	float_status *fpst; \
2962	float_status scratch_fpst; \
2963	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
2964	if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) { \
2965	continue; \
2966	} \
2967	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
2968	if (!(mask & 1)) { \
2969	/* We need the result but without updating flags */ \
2970	scratch_fpst = *fpst; \
2971	fpst = &scratch_fpst; \
2972	} \
2973	r = n[H##ESIZE(e)]; \
2974	if (CHS) { \
2975	r = TYPE##_chs(r); \
2976	} \
2977	r = TYPE##_muladd(r, m[H##ESIZE(e)], d[H##ESIZE(e)], \
2978	0, fpst); \
2979	mergemask(&d[H##ESIZE(e)], r, mask); \
2980	} \
2981	mve_advance_vpt(env); \
2982	}
2983
2984	DO_VFMA(vfmah, 2, float16, false)
2985	DO_VFMA(vfmas, 4, float32, false)
2986	DO_VFMA(vfmsh, 2, float16, true)
2987	DO_VFMA(vfmss, 4, float32, true)
2988
2989	#define DO_VCMLA(OP, ESIZE, TYPE, ROT, FN) \
2990	void HELPER(glue(mve_, OP))(CPUARMState *env, \
2991	void vd, void vn, void *vm) \
2992	{ \
2993	TYPE d = vd, n = vn, *m = vm; \
2994	TYPE r0, r1, e1, e2, e3, e4; \
2995	uint16_t mask = mve_element_mask(env); \
2996	unsigned e; \
2997	float_status fpst0, fpst1; \
2998	float_status scratch_fpst; \
2999	/* We loop through pairs of elements at a time */ \
3000	for (e = 0; e < 16 / ESIZE; e += 2, mask >>= ESIZE * 2) { \
3001	if ((mask & MAKE_64BIT_MASK(0, ESIZE * 2)) == 0) { \
3002	continue; \
3003	} \
3004	fpst0 = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3005	fpst1 = fpst0; \
3006	if (!(mask & 1)) { \
3007	scratch_fpst = *fpst0; \
3008	fpst0 = &scratch_fpst; \
3009	} \
3010	if (!(mask & (1 << ESIZE))) { \
3011	scratch_fpst = *fpst1; \
3012	fpst1 = &scratch_fpst; \
3013	} \
3014	switch (ROT) { \
3015	case 0: \
3016	e1 = m[H##ESIZE(e)]; \
3017	e2 = n[H##ESIZE(e)]; \
3018	e3 = m[H##ESIZE(e + 1)]; \
3019	e4 = n[H##ESIZE(e)]; \
3020	break; \
3021	case 1: \
3022	e1 = TYPE##_chs(m[H##ESIZE(e + 1)]); \
3023	e2 = n[H##ESIZE(e + 1)]; \
3024	e3 = m[H##ESIZE(e)]; \
3025	e4 = n[H##ESIZE(e + 1)]; \
3026	break; \
3027	case 2: \
3028	e1 = TYPE##_chs(m[H##ESIZE(e)]); \
3029	e2 = n[H##ESIZE(e)]; \
3030	e3 = TYPE##_chs(m[H##ESIZE(e + 1)]); \
3031	e4 = n[H##ESIZE(e)]; \
3032	break; \
3033	case 3: \
3034	e1 = m[H##ESIZE(e + 1)]; \
3035	e2 = n[H##ESIZE(e + 1)]; \
3036	e3 = TYPE##_chs(m[H##ESIZE(e)]); \
3037	e4 = n[H##ESIZE(e + 1)]; \
3038	break; \
3039	default: \
3040	g_assert_not_reached(); \
3041	} \
3042	r0 = FN(e2, e1, d[H##ESIZE(e)], fpst0); \
3043	r1 = FN(e4, e3, d[H##ESIZE(e + 1)], fpst1); \
3044	mergemask(&d[H##ESIZE(e)], r0, mask); \
3045	mergemask(&d[H##ESIZE(e + 1)], r1, mask >> ESIZE); \
3046	} \
3047	mve_advance_vpt(env); \
3048	}
3049
3050	#define DO_VCMULH(N, M, D, S) float16_mul(N, M, S)
3051	#define DO_VCMULS(N, M, D, S) float32_mul(N, M, S)
3052
3053	#define DO_VCMLAH(N, M, D, S) float16_muladd(N, M, D, 0, S)
3054	#define DO_VCMLAS(N, M, D, S) float32_muladd(N, M, D, 0, S)
3055
3056	DO_VCMLA(vcmul0h, 2, float16, 0, DO_VCMULH)
3057	DO_VCMLA(vcmul0s, 4, float32, 0, DO_VCMULS)
3058	DO_VCMLA(vcmul90h, 2, float16, 1, DO_VCMULH)
3059	DO_VCMLA(vcmul90s, 4, float32, 1, DO_VCMULS)
3060	DO_VCMLA(vcmul180h, 2, float16, 2, DO_VCMULH)
3061	DO_VCMLA(vcmul180s, 4, float32, 2, DO_VCMULS)
3062	DO_VCMLA(vcmul270h, 2, float16, 3, DO_VCMULH)
3063	DO_VCMLA(vcmul270s, 4, float32, 3, DO_VCMULS)
3064
3065	DO_VCMLA(vcmla0h, 2, float16, 0, DO_VCMLAH)
3066	DO_VCMLA(vcmla0s, 4, float32, 0, DO_VCMLAS)
3067	DO_VCMLA(vcmla90h, 2, float16, 1, DO_VCMLAH)
3068	DO_VCMLA(vcmla90s, 4, float32, 1, DO_VCMLAS)
3069	DO_VCMLA(vcmla180h, 2, float16, 2, DO_VCMLAH)
3070	DO_VCMLA(vcmla180s, 4, float32, 2, DO_VCMLAS)
3071	DO_VCMLA(vcmla270h, 2, float16, 3, DO_VCMLAH)
3072	DO_VCMLA(vcmla270s, 4, float32, 3, DO_VCMLAS)
3073
3074	#define DO_2OP_FP_SCALAR(OP, ESIZE, TYPE, FN) \
3075	void HELPER(glue(mve_, OP))(CPUARMState *env, \
3076	void vd, void vn, uint32_t rm) \
3077	{ \
3078	TYPE d = vd, n = vn; \
3079	TYPE r, m = rm; \
3080	uint16_t mask = mve_element_mask(env); \
3081	unsigned e; \
3082	float_status *fpst; \
3083	float_status scratch_fpst; \
3084	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
3085	if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) { \
3086	continue; \
3087	} \
3088	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3089	if (!(mask & 1)) { \
3090	/* We need the result but without updating flags */ \
3091	scratch_fpst = *fpst; \
3092	fpst = &scratch_fpst; \
3093	} \
3094	r = FN(n[H##ESIZE(e)], m, fpst); \
3095	mergemask(&d[H##ESIZE(e)], r, mask); \
3096	} \
3097	mve_advance_vpt(env); \
3098	}
3099
3100	#define DO_2OP_FP_SCALAR_ALL(OP, FN) \
3101	DO_2OP_FP_SCALAR(OP##h, 2, float16, float16_##FN) \
3102	DO_2OP_FP_SCALAR(OP##s, 4, float32, float32_##FN)
3103
3104	DO_2OP_FP_SCALAR_ALL(vfadd_scalar, add)
3105	DO_2OP_FP_SCALAR_ALL(vfsub_scalar, sub)
3106	DO_2OP_FP_SCALAR_ALL(vfmul_scalar, mul)
3107
3108	#define DO_2OP_FP_ACC_SCALAR(OP, ESIZE, TYPE, FN) \
3109	void HELPER(glue(mve_, OP))(CPUARMState *env, \
3110	void vd, void vn, uint32_t rm) \
3111	{ \
3112	TYPE d = vd, n = vn; \
3113	TYPE r, m = rm; \
3114	uint16_t mask = mve_element_mask(env); \
3115	unsigned e; \
3116	float_status *fpst; \
3117	float_status scratch_fpst; \
3118	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
3119	if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) { \
3120	continue; \
3121	} \
3122	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3123	if (!(mask & 1)) { \
3124	/* We need the result but without updating flags */ \
3125	scratch_fpst = *fpst; \
3126	fpst = &scratch_fpst; \
3127	} \
3128	r = FN(n[H##ESIZE(e)], m, d[H##ESIZE(e)], 0, fpst); \
3129	mergemask(&d[H##ESIZE(e)], r, mask); \
3130	} \
3131	mve_advance_vpt(env); \
3132	}
3133
3134	/* VFMAS is vector * vector + scalar, so swap op2 and op3 */
3135	#define DO_VFMAS_SCALARH(N, M, D, F, S) float16_muladd(N, D, M, F, S)
3136	#define DO_VFMAS_SCALARS(N, M, D, F, S) float32_muladd(N, D, M, F, S)
3137
3138	/* VFMA is vector * scalar + vector */
3139	DO_2OP_FP_ACC_SCALAR(vfma_scalarh, 2, float16, float16_muladd)
3140	DO_2OP_FP_ACC_SCALAR(vfma_scalars, 4, float32, float32_muladd)
3141	DO_2OP_FP_ACC_SCALAR(vfmas_scalarh, 2, float16, DO_VFMAS_SCALARH)
3142	DO_2OP_FP_ACC_SCALAR(vfmas_scalars, 4, float32, DO_VFMAS_SCALARS)
3143
3144	/* Floating point max/min across vector. */
3145	#define DO_FP_VMAXMINV(OP, ESIZE, TYPE, ABS, FN) \
3146	uint32_t HELPER(glue(mve_, OP))(CPUARMState env, void vm, \
3147	uint32_t ra_in) \
3148	{ \
3149	uint16_t mask = mve_element_mask(env); \
3150	unsigned e; \
3151	TYPE *m = vm; \
3152	TYPE ra = (TYPE)ra_in; \
3153	float_status *fpst = \
3154	&env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3155	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
3156	if (mask & 1) { \
3157	TYPE v = m[H##ESIZE(e)]; \
3158	if (TYPE##_is_signaling_nan(ra, fpst)) { \
3159	ra = TYPE##_silence_nan(ra, fpst); \
3160	float_raise(float_flag_invalid, fpst); \
3161	} \
3162	if (TYPE##_is_signaling_nan(v, fpst)) { \
3163	v = TYPE##_silence_nan(v, fpst); \
3164	float_raise(float_flag_invalid, fpst); \
3165	} \
3166	if (ABS) { \
3167	v = TYPE##_abs(v); \
3168	} \
3169	ra = FN(ra, v, fpst); \
3170	} \
3171	} \
3172	mve_advance_vpt(env); \
3173	return ra; \
3174	} \
3175
3176	#define NOP(X) (X)
3177
3178	DO_FP_VMAXMINV(vmaxnmvh, 2, float16, false, float16_maxnum)
3179	DO_FP_VMAXMINV(vmaxnmvs, 4, float32, false, float32_maxnum)
3180	DO_FP_VMAXMINV(vminnmvh, 2, float16, false, float16_minnum)
3181	DO_FP_VMAXMINV(vminnmvs, 4, float32, false, float32_minnum)
3182	DO_FP_VMAXMINV(vmaxnmavh, 2, float16, true, float16_maxnum)
3183	DO_FP_VMAXMINV(vmaxnmavs, 4, float32, true, float32_maxnum)
3184	DO_FP_VMAXMINV(vminnmavh, 2, float16, true, float16_minnum)
3185	DO_FP_VMAXMINV(vminnmavs, 4, float32, true, float32_minnum)
3186
3187	/* FP compares; note that all comparisons signal InvalidOp for QNaNs */
3188	#define DO_VCMP_FP(OP, ESIZE, TYPE, FN) \
3189	void HELPER(glue(mve_, OP))(CPUARMState env, void vn, void *vm) \
3190	{ \
3191	TYPE n = vn, m = vm; \
3192	uint16_t mask = mve_element_mask(env); \
3193	uint16_t eci_mask = mve_eci_mask(env); \
3194	uint16_t beatpred = 0; \
3195	uint16_t emask = MAKE_64BIT_MASK(0, ESIZE); \
3196	unsigned e; \
3197	float_status *fpst; \
3198	float_status scratch_fpst; \
3199	bool r; \
3200	for (e = 0; e < 16 / ESIZE; e++, emask <<= ESIZE) { \
3201	if ((mask & emask) == 0) { \
3202	continue; \
3203	} \
3204	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3205	if (!(mask & (1 << (e * ESIZE)))) { \
3206	/* We need the result but without updating flags */ \
3207	scratch_fpst = *fpst; \
3208	fpst = &scratch_fpst; \
3209	} \
3210	r = FN(n[H##ESIZE(e)], m[H##ESIZE(e)], fpst); \
3211	/* Comparison sets 0/1 bits for each byte in the element */ \
3212	beatpred \|= r * emask; \
3213	} \
3214	beatpred &= mask; \
3215	env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) \| \
3216	(beatpred & eci_mask); \
3217	mve_advance_vpt(env); \
3218	}
3219
3220	#define DO_VCMP_FP_SCALAR(OP, ESIZE, TYPE, FN) \
3221	void HELPER(glue(mve_, OP))(CPUARMState env, void vn, \
3222	uint32_t rm) \
3223	{ \
3224	TYPE *n = vn; \
3225	uint16_t mask = mve_element_mask(env); \
3226	uint16_t eci_mask = mve_eci_mask(env); \
3227	uint16_t beatpred = 0; \
3228	uint16_t emask = MAKE_64BIT_MASK(0, ESIZE); \
3229	unsigned e; \
3230	float_status *fpst; \
3231	float_status scratch_fpst; \
3232	bool r; \
3233	for (e = 0; e < 16 / ESIZE; e++, emask <<= ESIZE) { \
3234	if ((mask & emask) == 0) { \
3235	continue; \
3236	} \
3237	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3238	if (!(mask & (1 << (e * ESIZE)))) { \
3239	/* We need the result but without updating flags */ \
3240	scratch_fpst = *fpst; \
3241	fpst = &scratch_fpst; \
3242	} \
3243	r = FN(n[H##ESIZE(e)], (TYPE)rm, fpst); \
3244	/* Comparison sets 0/1 bits for each byte in the element */ \
3245	beatpred \|= r * emask; \
3246	} \
3247	beatpred &= mask; \
3248	env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) \| \
3249	(beatpred & eci_mask); \
3250	mve_advance_vpt(env); \
3251	}
3252
3253	#define DO_VCMP_FP_BOTH(VOP, SOP, ESIZE, TYPE, FN) \
3254	DO_VCMP_FP(VOP, ESIZE, TYPE, FN) \
3255	DO_VCMP_FP_SCALAR(SOP, ESIZE, TYPE, FN)
3256
3257	/*
3258	* Some care is needed here to get the correct result for the unordered case.
3259	* Architecturally EQ, GE and GT are defined to be false for unordered, but
3260	* the NE, LT and LE comparisons are defined as simple logical inverses of
3261	* EQ, GE and GT and so they must return true for unordered. The softfloat
3262	* comparison functions float*_{eq,le,lt} all return false for unordered.
3263	*/
3264	#define DO_GE16(X, Y, S) float16_le(Y, X, S)
3265	#define DO_GE32(X, Y, S) float32_le(Y, X, S)
3266	#define DO_GT16(X, Y, S) float16_lt(Y, X, S)
3267	#define DO_GT32(X, Y, S) float32_lt(Y, X, S)
3268
3269	DO_VCMP_FP_BOTH(vfcmpeqh, vfcmpeq_scalarh, 2, float16, float16_eq)
3270	DO_VCMP_FP_BOTH(vfcmpeqs, vfcmpeq_scalars, 4, float32, float32_eq)
3271
3272	DO_VCMP_FP_BOTH(vfcmpneh, vfcmpne_scalarh, 2, float16, !float16_eq)
3273	DO_VCMP_FP_BOTH(vfcmpnes, vfcmpne_scalars, 4, float32, !float32_eq)
3274
3275	DO_VCMP_FP_BOTH(vfcmpgeh, vfcmpge_scalarh, 2, float16, DO_GE16)
3276	DO_VCMP_FP_BOTH(vfcmpges, vfcmpge_scalars, 4, float32, DO_GE32)
3277
3278	DO_VCMP_FP_BOTH(vfcmplth, vfcmplt_scalarh, 2, float16, !DO_GE16)
3279	DO_VCMP_FP_BOTH(vfcmplts, vfcmplt_scalars, 4, float32, !DO_GE32)
3280
3281	DO_VCMP_FP_BOTH(vfcmpgth, vfcmpgt_scalarh, 2, float16, DO_GT16)
3282	DO_VCMP_FP_BOTH(vfcmpgts, vfcmpgt_scalars, 4, float32, DO_GT32)
3283
3284	DO_VCMP_FP_BOTH(vfcmpleh, vfcmple_scalarh, 2, float16, !DO_GT16)
3285	DO_VCMP_FP_BOTH(vfcmples, vfcmple_scalars, 4, float32, !DO_GT32)
3286
3287	#define DO_VCVT_FIXED(OP, ESIZE, TYPE, FN) \
3288	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vm, \
3289	uint32_t shift) \
3290	{ \
3291	TYPE d = vd, m = vm; \
3292	TYPE r; \
3293	uint16_t mask = mve_element_mask(env); \
3294	unsigned e; \
3295	float_status *fpst; \
3296	float_status scratch_fpst; \
3297	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
3298	if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) { \
3299	continue; \
3300	} \
3301	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3302	if (!(mask & 1)) { \
3303	/* We need the result but without updating flags */ \
3304	scratch_fpst = *fpst; \
3305	fpst = &scratch_fpst; \
3306	} \
3307	r = FN(m[H##ESIZE(e)], shift, fpst); \
3308	mergemask(&d[H##ESIZE(e)], r, mask); \
3309	} \
3310	mve_advance_vpt(env); \
3311	}
3312
3313	DO_VCVT_FIXED(vcvt_sh, 2, int16_t, helper_vfp_shtoh)
3314	DO_VCVT_FIXED(vcvt_uh, 2, uint16_t, helper_vfp_uhtoh)
3315	DO_VCVT_FIXED(vcvt_hs, 2, int16_t, helper_vfp_toshh_round_to_zero)
3316	DO_VCVT_FIXED(vcvt_hu, 2, uint16_t, helper_vfp_touhh_round_to_zero)
3317	DO_VCVT_FIXED(vcvt_sf, 4, int32_t, helper_vfp_sltos)
3318	DO_VCVT_FIXED(vcvt_uf, 4, uint32_t, helper_vfp_ultos)
3319	DO_VCVT_FIXED(vcvt_fs, 4, int32_t, helper_vfp_tosls_round_to_zero)
3320	DO_VCVT_FIXED(vcvt_fu, 4, uint32_t, helper_vfp_touls_round_to_zero)
3321
3322	/* VCVT with specified rmode */
3323	#define DO_VCVT_RMODE(OP, ESIZE, TYPE, FN) \
3324	void HELPER(glue(mve_, OP))(CPUARMState *env, \
3325	void vd, void vm, uint32_t rmode) \
3326	{ \
3327	TYPE d = vd, m = vm; \
3328	TYPE r; \
3329	uint16_t mask = mve_element_mask(env); \
3330	unsigned e; \
3331	float_status *fpst; \
3332	float_status scratch_fpst; \
3333	float_status *base_fpst = \
3334	&env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3335	uint32_t prev_rmode = get_float_rounding_mode(base_fpst); \
3336	set_float_rounding_mode(rmode, base_fpst); \
3337	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
3338	if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) { \
3339	continue; \
3340	} \
3341	fpst = base_fpst; \
3342	if (!(mask & 1)) { \
3343	/* We need the result but without updating flags */ \
3344	scratch_fpst = *fpst; \
3345	fpst = &scratch_fpst; \
3346	} \
3347	r = FN(m[H##ESIZE(e)], 0, fpst); \
3348	mergemask(&d[H##ESIZE(e)], r, mask); \
3349	} \
3350	set_float_rounding_mode(prev_rmode, base_fpst); \
3351	mve_advance_vpt(env); \
3352	}
3353
3354	DO_VCVT_RMODE(vcvt_rm_sh, 2, uint16_t, helper_vfp_toshh)
3355	DO_VCVT_RMODE(vcvt_rm_uh, 2, uint16_t, helper_vfp_touhh)
3356	DO_VCVT_RMODE(vcvt_rm_ss, 4, uint32_t, helper_vfp_tosls)
3357	DO_VCVT_RMODE(vcvt_rm_us, 4, uint32_t, helper_vfp_touls)
3358
3359	#define DO_VRINT_RM_H(M, F, S) helper_rinth(M, S)
3360	#define DO_VRINT_RM_S(M, F, S) helper_rints(M, S)
3361
3362	DO_VCVT_RMODE(vrint_rm_h, 2, uint16_t, DO_VRINT_RM_H)
3363	DO_VCVT_RMODE(vrint_rm_s, 4, uint32_t, DO_VRINT_RM_S)
3364
3365	/*
3366	* VCVT between halfprec and singleprec. As usual for halfprec
3367	* conversions, FZ16 is ignored and AHP is observed.
3368	*/
3369	static void do_vcvt_sh(CPUARMState env, void vd, void *vm, int top)
3370	{
3371	uint16_t *d = vd;
3372	uint32_t *m = vm;
3373	uint16_t r;
3374	uint16_t mask = mve_element_mask(env);
3375	bool ieee = !(env->vfp.fpcr & FPCR_AHP);
3376	unsigned e;
3377	float_status *fpst;
3378	float_status scratch_fpst;
3379	float_status *base_fpst = &env->vfp.fp_status[FPST_STD];
3380	bool old_fz = get_flush_to_zero(base_fpst);
3381	set_flush_to_zero(false, base_fpst);
3382	for (e = 0; e < 16 / 4; e++, mask >>= 4) {
3383	if ((mask & MAKE_64BIT_MASK(0, 4)) == 0) {
3384	continue;
3385	}
3386	fpst = base_fpst;
3387	if (!(mask & 1)) {
3388	/* We need the result but without updating flags */
3389	scratch_fpst = *fpst;
3390	fpst = &scratch_fpst;
3391	}
3392	r = float32_to_float16(m[H4(e)], ieee, fpst);
3393	mergemask(&d[H2(e * 2 + top)], r, mask >> (top * 2));
3394	}
3395	set_flush_to_zero(old_fz, base_fpst);
3396	mve_advance_vpt(env);
3397	}
3398
3399	static void do_vcvt_hs(CPUARMState env, void vd, void *vm, int top)
3400	{
3401	uint32_t *d = vd;
3402	uint16_t *m = vm;
3403	uint32_t r;
3404	uint16_t mask = mve_element_mask(env);
3405	bool ieee = !(env->vfp.fpcr & FPCR_AHP);
3406	unsigned e;
3407	float_status *fpst;
3408	float_status scratch_fpst;
3409	float_status *base_fpst = &env->vfp.fp_status[FPST_STD];
3410	bool old_fiz = get_flush_inputs_to_zero(base_fpst);
3411	set_flush_inputs_to_zero(false, base_fpst);
3412	for (e = 0; e < 16 / 4; e++, mask >>= 4) {
3413	if ((mask & MAKE_64BIT_MASK(0, 4)) == 0) {
3414	continue;
3415	}
3416	fpst = base_fpst;
3417	if (!(mask & (1 << (top * 2)))) {
3418	/* We need the result but without updating flags */
3419	scratch_fpst = *fpst;
3420	fpst = &scratch_fpst;
3421	}
3422	r = float16_to_float32(m[H2(e * 2 + top)], ieee, fpst);
3423	mergemask(&d[H4(e)], r, mask);
3424	}
3425	set_flush_inputs_to_zero(old_fiz, base_fpst);
3426	mve_advance_vpt(env);
3427	}
3428
3429	void HELPER(mve_vcvtb_sh)(CPUARMState env, void vd, void *vm)
3430	{
3431	do_vcvt_sh(env, vd, vm, 0);
3432	}
3433	void HELPER(mve_vcvtt_sh)(CPUARMState env, void vd, void *vm)
3434	{
3435	do_vcvt_sh(env, vd, vm, 1);
3436	}
3437	void HELPER(mve_vcvtb_hs)(CPUARMState env, void vd, void *vm)
3438	{
3439	do_vcvt_hs(env, vd, vm, 0);
3440	}
3441	void HELPER(mve_vcvtt_hs)(CPUARMState env, void vd, void *vm)
3442	{
3443	do_vcvt_hs(env, vd, vm, 1);
3444	}
3445
3446	#define DO_1OP_FP(OP, ESIZE, TYPE, FN) \
3447	void HELPER(glue(mve_, OP))(CPUARMState env, void vd, void *vm) \
3448	{ \
3449	TYPE d = vd, m = vm; \
3450	TYPE r; \
3451	uint16_t mask = mve_element_mask(env); \
3452	unsigned e; \
3453	float_status *fpst; \
3454	float_status scratch_fpst; \
3455	for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) { \
3456	if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) { \
3457	continue; \
3458	} \
3459	fpst = &env->vfp.fp_status[ESIZE == 2 ? FPST_STD_F16 : FPST_STD]; \
3460	if (!(mask & 1)) { \
3461	/* We need the result but without updating flags */ \
3462	scratch_fpst = *fpst; \
3463	fpst = &scratch_fpst; \
3464	} \
3465	r = FN(m[H##ESIZE(e)], fpst); \
3466	mergemask(&d[H##ESIZE(e)], r, mask); \
3467	} \
3468	mve_advance_vpt(env); \
3469	}
3470
3471	DO_1OP_FP(vrintx_h, 2, float16, float16_round_to_int)
3472	DO_1OP_FP(vrintx_s, 4, float32, float32_round_to_int)