translate-sve.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 8,976 lines 299 KB

Raw

1	/*
2	* AArch64 SVE translation
3	*
4	* Copyright (c) 2018 Linaro, Ltd
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#include "qemu/osdep.h"
21	#include "cpu.h"
22	#include "helper-sme.h"
23	#include "helper-sve.h"
24	#include "helper-fp8.h"
25	#include "translate.h"
26	#include "translate-a64.h"
27	#include "tcg/tcg-op.h"
28	#include "fpu/softfloat.h"
29
30
31	typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
32	TCGv_i64, uint32_t, uint32_t);
33
34	typedef void gen_helper_gvec_flags_3(TCGv_i32, TCGv_ptr, TCGv_ptr,
35	TCGv_ptr, TCGv_i32);
36	typedef void gen_helper_gvec_flags_4(TCGv_i32, TCGv_ptr, TCGv_ptr,
37	TCGv_ptr, TCGv_ptr, TCGv_i32);
38
39	typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i64);
40	typedef void gen_helper_gvec_mem_scatter(TCGv_env, TCGv_ptr, TCGv_ptr,
41	TCGv_ptr, TCGv_i64, TCGv_i64);
42
43	/*
44	* Helpers for extracting complex instruction fields.
45	*/
46
47	/* See e.g. ASR (immediate, predicated).
48	* Returns -1 for unallocated encoding; diagnose later.
49	*/
50	static int tszimm_esz(DisasContext *s, int x)
51	{
52	x >>= 3; /* discard imm3 */
53	return 31 - clz32(x);
54	}
55
56	static int tszimm_shr(DisasContext *s, int x)
57	{
58	/*
59	* We won't use the tszimm_shr() value if tszimm_esz() returns -1 (the
60	* trans function will check for esz < 0), so we can return any
61	* value we like from here in that case as long as we avoid UB.
62	*/
63	int esz = tszimm_esz(s, x);
64	if (esz < 0) {
65	return esz;
66	}
67	return (16 << esz) - x;
68	}
69
70	/* See e.g. LSL (immediate, predicated). */
71	static int tszimm_shl(DisasContext *s, int x)
72	{
73	/* As with tszimm_shr(), value will be unused if esz < 0 */
74	int esz = tszimm_esz(s, x);
75	if (esz < 0) {
76	return esz;
77	}
78	return x - (8 << esz);
79	}
80
81	/* The SH bit is in bit 8. Extract the low 8 and shift. */
82	static inline int expand_imm_sh8s(DisasContext *s, int x)
83	{
84	return (int8_t)x << (x & 0x100 ? 8 : 0);
85	}
86
87	static inline int expand_imm_sh8u(DisasContext *s, int x)
88	{
89	return (uint8_t)x << (x & 0x100 ? 8 : 0);
90	}
91
92	/* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
93	* with unsigned data. C.f. SVE Memory Contiguous Load Group.
94	*/
95	static inline int msz_dtype(DisasContext *s, int msz)
96	{
97	static const uint8_t dtype[5] = { 0, 5, 10, 15, 18 };
98	return dtype[msz];
99	}
100
101	/*
102	* Include the generated decoder.
103	*/
104
105	#include "decode-sve.c.inc"
106
107	/*
108	* Implement all of the translator functions referenced by the decoder.
109	*/
110
111	/* Invoke an out-of-line helper on 2 Zregs. */
112	static bool gen_gvec_ool_zz(DisasContext s, gen_helper_gvec_2 fn,
113	int rd, int rn, int data)
114	{
115	if (fn == NULL) {
116	return false;
117	}
118	if (sve_access_check(s)) {
119	unsigned vsz = vec_full_reg_size(s);
120	tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
121	vec_full_reg_offset(s, rn),
122	vsz, vsz, data, fn);
123	}
124	return true;
125	}
126
127	static bool gen_gvec_fpst_zz(DisasContext s, gen_helper_gvec_2_ptr fn,
128	int rd, int rn, int data,
129	ARMFPStatusFlavour flavour)
130	{
131	if (fn == NULL) {
132	return false;
133	}
134	if (sve_access_check(s)) {
135	unsigned vsz = vec_full_reg_size(s);
136	TCGv_ptr status = fpstatus_ptr(flavour);
137
138	tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd),
139	vec_full_reg_offset(s, rn),
140	status, vsz, vsz, data, fn);
141	}
142	return true;
143	}
144
145	static bool gen_gvec_fpst_ah_arg_zz(DisasContext s, gen_helper_gvec_2_ptr fn,
146	arg_rr_esz *a, int data)
147	{
148	return gen_gvec_fpst_zz(s, fn, a->rd, a->rn, data,
149	select_ah_fpst(s, a->esz));
150	}
151
152	/* Invoke an out-of-line helper on 3 Zregs. */
153	static bool gen_gvec_ool_zzz(DisasContext s, gen_helper_gvec_3 fn,
154	int rd, int rn, int rm, int data)
155	{
156	if (fn == NULL) {
157	return false;
158	}
159	if (sve_access_check(s)) {
160	unsigned vsz = vec_full_reg_size(s);
161	tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
162	vec_full_reg_offset(s, rn),
163	vec_full_reg_offset(s, rm),
164	vsz, vsz, data, fn);
165	}
166	return true;
167	}
168
169	static bool gen_gvec_ool_arg_zzz(DisasContext s, gen_helper_gvec_3 fn,
170	arg_rrr_esz *a, int data)
171	{
172	return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data);
173	}
174
175	/* Invoke an out-of-line helper on 3 Zregs, plus float_status. */
176	static bool gen_gvec_fpst_zzz(DisasContext s, gen_helper_gvec_3_ptr fn,
177	int rd, int rn, int rm,
178	int data, ARMFPStatusFlavour flavour)
179	{
180	if (fn == NULL) {
181	return false;
182	}
183	if (sve_access_check(s)) {
184	unsigned vsz = vec_full_reg_size(s);
185	TCGv_ptr status = fpstatus_ptr(flavour);
186
187	tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
188	vec_full_reg_offset(s, rn),
189	vec_full_reg_offset(s, rm),
190	status, vsz, vsz, data, fn);
191	}
192	return true;
193	}
194
195	static bool gen_gvec_fpst_arg_zzz(DisasContext s, gen_helper_gvec_3_ptr fn,
196	arg_rrr_esz *a, int data)
197	{
198	/* These insns use MO_8 to encode BFloat16 */
199	if (a->esz == MO_8 && !dc_isar_feature(aa64_sve_b16b16, s)) {
200	return false;
201	}
202	return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data,
203	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
204	}
205
206	static bool gen_gvec_fpst_ah_arg_zzz(DisasContext s, gen_helper_gvec_3_ptr fn,
207	arg_rrr_esz *a, int data)
208	{
209	return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data,
210	select_ah_fpst(s, a->esz));
211	}
212
213	/* Invoke an out-of-line helper on 4 Zregs. */
214	static bool gen_gvec_ool_zzzz(DisasContext s, gen_helper_gvec_4 fn,
215	int rd, int rn, int rm, int ra, int data)
216	{
217	if (fn == NULL) {
218	return false;
219	}
220	if (sve_access_check(s)) {
221	unsigned vsz = vec_full_reg_size(s);
222	tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
223	vec_full_reg_offset(s, rn),
224	vec_full_reg_offset(s, rm),
225	vec_full_reg_offset(s, ra),
226	vsz, vsz, data, fn);
227	}
228	return true;
229	}
230
231	static bool gen_gvec_ool_arg_zzzz(DisasContext s, gen_helper_gvec_4 fn,
232	arg_rrrr_esz *a, int data)
233	{
234	return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
235	}
236
237	static bool gen_gvec_ool_arg_zzxz(DisasContext s, gen_helper_gvec_4 fn,
238	arg_rrxr_esz *a)
239	{
240	return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index);
241	}
242
243	/* Invoke an out-of-line helper on 4 Zregs, plus a pointer. */
244	static bool gen_gvec_ptr_zzzz(DisasContext s, gen_helper_gvec_4_ptr fn,
245	int rd, int rn, int rm, int ra,
246	int data, TCGv_ptr ptr)
247	{
248	if (fn == NULL) {
249	return false;
250	}
251	if (sve_access_check(s)) {
252	unsigned vsz = vec_full_reg_size(s);
253	tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
254	vec_full_reg_offset(s, rn),
255	vec_full_reg_offset(s, rm),
256	vec_full_reg_offset(s, ra),
257	ptr, vsz, vsz, data, fn);
258	}
259	return true;
260	}
261
262	static bool gen_gvec_fpst_zzzz(DisasContext s, gen_helper_gvec_4_ptr fn,
263	int rd, int rn, int rm, int ra,
264	int data, ARMFPStatusFlavour flavour)
265	{
266	TCGv_ptr status = fpstatus_ptr(flavour);
267	bool ret = gen_gvec_ptr_zzzz(s, fn, rd, rn, rm, ra, data, status);
268	return ret;
269	}
270
271	static bool gen_gvec_env_zzzz(DisasContext s, gen_helper_gvec_4_ptr fn,
272	int rd, int rn, int rm, int ra,
273	int data)
274	{
275	return gen_gvec_ptr_zzzz(s, fn, rd, rn, rm, ra, data, tcg_env);
276	}
277
278	static bool gen_gvec_env_arg_zzzz(DisasContext s, gen_helper_gvec_4_ptr fn,
279	arg_rrrr_esz *a, int data)
280	{
281	return gen_gvec_env_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
282	}
283
284	static bool gen_gvec_env_arg_zzxz(DisasContext s, gen_helper_gvec_4_ptr fn,
285	arg_rrxr_esz *a)
286	{
287	return gen_gvec_env_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index);
288	}
289
290	/* Invoke an out-of-line helper on 4 Zregs, 1 Preg, plus fpst. */
291	static bool gen_gvec_fpst_zzzzp(DisasContext s, gen_helper_gvec_5_ptr fn,
292	int rd, int rn, int rm, int ra, int pg,
293	int data, ARMFPStatusFlavour flavour)
294	{
295	if (fn == NULL) {
296	return false;
297	}
298	if (sve_access_check(s)) {
299	unsigned vsz = vec_full_reg_size(s);
300	TCGv_ptr status = fpstatus_ptr(flavour);
301
302	tcg_gen_gvec_5_ptr(vec_full_reg_offset(s, rd),
303	vec_full_reg_offset(s, rn),
304	vec_full_reg_offset(s, rm),
305	vec_full_reg_offset(s, ra),
306	pred_full_reg_offset(s, pg),
307	status, vsz, vsz, data, fn);
308	}
309	return true;
310	}
311
312	/* Invoke an out-of-line helper on 2 Zregs and a predicate. */
313	static bool gen_gvec_ool_zzp(DisasContext s, gen_helper_gvec_3 fn,
314	int rd, int rn, int pg, int data)
315	{
316	if (fn == NULL) {
317	return false;
318	}
319	if (sve_access_check(s)) {
320	unsigned vsz = vec_full_reg_size(s);
321	tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
322	vec_full_reg_offset(s, rn),
323	pred_full_reg_offset(s, pg),
324	vsz, vsz, data, fn);
325	}
326	return true;
327	}
328
329	static bool gen_gvec_ool_arg_zpz(DisasContext s, gen_helper_gvec_3 fn,
330	arg_rpr_esz *a, int data)
331	{
332	return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, data);
333	}
334
335	static bool gen_gvec_ool_arg_zpzi(DisasContext s, gen_helper_gvec_3 fn,
336	arg_rpri_esz *a)
337	{
338	return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm);
339	}
340
341	static bool gen_gvec_fpst_zzp(DisasContext s, gen_helper_gvec_3_ptr fn,
342	int rd, int rn, int pg, int data,
343	ARMFPStatusFlavour flavour)
344	{
345	if (fn == NULL) {
346	return false;
347	}
348	if (sve_access_check(s)) {
349	unsigned vsz = vec_full_reg_size(s);
350	TCGv_ptr status = fpstatus_ptr(flavour);
351
352	tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
353	vec_full_reg_offset(s, rn),
354	pred_full_reg_offset(s, pg),
355	status, vsz, vsz, data, fn);
356	}
357	return true;
358	}
359
360	static bool gen_gvec_fpst_arg_zpz(DisasContext s, gen_helper_gvec_3_ptr fn,
361	arg_rpr_esz *a, int data,
362	ARMFPStatusFlavour flavour)
363	{
364	return gen_gvec_fpst_zzp(s, fn, a->rd, a->rn, a->pg, data, flavour);
365	}
366
367	/* Invoke an out-of-line helper on 3 Zregs and a predicate. */
368	static bool gen_gvec_ool_zzzp(DisasContext s, gen_helper_gvec_4 fn,
369	int rd, int rn, int rm, int pg, int data)
370	{
371	if (fn == NULL) {
372	return false;
373	}
374	if (sve_access_check(s)) {
375	unsigned vsz = vec_full_reg_size(s);
376	tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
377	vec_full_reg_offset(s, rn),
378	vec_full_reg_offset(s, rm),
379	pred_full_reg_offset(s, pg),
380	vsz, vsz, data, fn);
381	}
382	return true;
383	}
384
385	static bool gen_gvec_ool_arg_zpzz(DisasContext s, gen_helper_gvec_4 fn,
386	arg_rprr_esz *a, int data)
387	{
388	return gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, data);
389	}
390
391	/* Invoke an out-of-line helper on 3 Zregs and a predicate. */
392	static bool gen_gvec_fpst_zzzp(DisasContext s, gen_helper_gvec_4_ptr fn,
393	int rd, int rn, int rm, int pg, int data,
394	ARMFPStatusFlavour flavour)
395	{
396	if (fn == NULL) {
397	return false;
398	}
399	if (sve_access_check(s)) {
400	unsigned vsz = vec_full_reg_size(s);
401	TCGv_ptr status = fpstatus_ptr(flavour);
402
403	tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
404	vec_full_reg_offset(s, rn),
405	vec_full_reg_offset(s, rm),
406	pred_full_reg_offset(s, pg),
407	status, vsz, vsz, data, fn);
408	}
409	return true;
410	}
411
412	static bool gen_gvec_fpst_arg_zpzz(DisasContext s, gen_helper_gvec_4_ptr fn,
413	arg_rprr_esz *a)
414	{
415	return gen_gvec_fpst_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0,
416	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
417	}
418
419	/* Invoke a vector expander on two Zregs and an immediate. */
420	static bool gen_gvec_fn_zzi(DisasContext s, GVecGen2iFn gvec_fn,
421	int esz, int rd, int rn, uint64_t imm)
422	{
423	if (gvec_fn == NULL) {
424	return false;
425	}
426	if (sve_access_check(s)) {
427	unsigned vsz = vec_full_reg_size(s);
428	gvec_fn(esz, vec_full_reg_offset(s, rd),
429	vec_full_reg_offset(s, rn), imm, vsz, vsz);
430	}
431	return true;
432	}
433
434	static bool gen_gvec_fn_arg_zzi(DisasContext s, GVecGen2iFn gvec_fn,
435	arg_rri_esz *a)
436	{
437	if (a->esz < 0) {
438	/* Invalid tsz encoding -- see tszimm_esz. */
439	return false;
440	}
441	return gen_gvec_fn_zzi(s, gvec_fn, a->esz, a->rd, a->rn, a->imm);
442	}
443
444	/* Invoke a vector expander on three Zregs. */
445	static bool gen_gvec_fn_zzz(DisasContext s, GVecGen3Fn gvec_fn,
446	int esz, int rd, int rn, int rm)
447	{
448	if (gvec_fn == NULL) {
449	return false;
450	}
451	if (sve_access_check(s)) {
452	unsigned vsz = vec_full_reg_size(s);
453	gvec_fn(esz, vec_full_reg_offset(s, rd),
454	vec_full_reg_offset(s, rn),
455	vec_full_reg_offset(s, rm), vsz, vsz);
456	}
457	return true;
458	}
459
460	static bool gen_gvec_fn_arg_zzz(DisasContext s, GVecGen3Fn fn,
461	arg_rrr_esz *a)
462	{
463	return gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm);
464	}
465
466	/* Invoke a vector expander on four Zregs. */
467	static bool gen_gvec_fn_arg_zzzz(DisasContext s, GVecGen4Fn gvec_fn,
468	arg_rrrr_esz *a)
469	{
470	if (gvec_fn == NULL) {
471	return false;
472	}
473	if (sve_access_check(s)) {
474	unsigned vsz = vec_full_reg_size(s);
475	gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
476	vec_full_reg_offset(s, a->rn),
477	vec_full_reg_offset(s, a->rm),
478	vec_full_reg_offset(s, a->ra), vsz, vsz);
479	}
480	return true;
481	}
482
483	/* Invoke a vector move on two Zregs. */
484	static bool do_mov_z(DisasContext *s, int rd, int rn)
485	{
486	if (sve_access_check(s)) {
487	unsigned vsz = vec_full_reg_size(s);
488	tcg_gen_gvec_mov(MO_8, vec_full_reg_offset(s, rd),
489	vec_full_reg_offset(s, rn), vsz, vsz);
490	}
491	return true;
492	}
493
494	/* Initialize a Zreg with replications of a 64-bit immediate. */
495	static void do_dupi_z(DisasContext *s, int rd, uint64_t word)
496	{
497	unsigned vsz = vec_full_reg_size(s);
498	tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), vsz, vsz, word);
499	}
500
501	/* Invoke a vector expander on three Pregs. */
502	static bool gen_gvec_fn_ppp(DisasContext s, GVecGen3Fn gvec_fn,
503	int rd, int rn, int rm)
504	{
505	if (sve_access_check(s)) {
506	unsigned psz = pred_gvec_reg_size(s);
507	gvec_fn(MO_64, pred_full_reg_offset(s, rd),
508	pred_full_reg_offset(s, rn),
509	pred_full_reg_offset(s, rm), psz, psz);
510	}
511	return true;
512	}
513
514	/* Invoke a vector move on two Pregs. */
515	static bool do_mov_p(DisasContext *s, int rd, int rn)
516	{
517	if (sve_access_check(s)) {
518	unsigned psz = pred_gvec_reg_size(s);
519	tcg_gen_gvec_mov(MO_8, pred_full_reg_offset(s, rd),
520	pred_full_reg_offset(s, rn), psz, psz);
521	}
522	return true;
523	}
524
525	/* Set the cpu flags as per a return from an SVE helper. */
526	static void do_pred_flags(TCGv_i32 t)
527	{
528	tcg_gen_mov_i32(cpu_NF, t);
529	tcg_gen_andi_i32(cpu_ZF, t, 2);
530	tcg_gen_andi_i32(cpu_CF, t, 1);
531	tcg_gen_movi_i32(cpu_VF, 0);
532	}
533
534	/* Subroutines computing the ARM PredTest psuedofunction. */
535	static void do_predtest1(TCGv_i64 d, TCGv_i64 g)
536	{
537	TCGv_i32 t = tcg_temp_new_i32();
538
539	gen_helper_sve_predtest1(t, d, g);
540	do_pred_flags(t);
541	}
542
543	static void do_predtest(DisasContext *s, int dofs, int gofs, int words)
544	{
545	TCGv_ptr dptr = tcg_temp_new_ptr();
546	TCGv_ptr gptr = tcg_temp_new_ptr();
547	TCGv_i32 t = tcg_temp_new_i32();
548
549	tcg_gen_addi_ptr(dptr, tcg_env, dofs);
550	tcg_gen_addi_ptr(gptr, tcg_env, gofs);
551
552	gen_helper_sve_predtest(t, dptr, gptr, tcg_constant_i32(words));
553
554	do_pred_flags(t);
555	}
556
557	/* For each element size, the bits within a predicate word that are active. */
558	const uint64_t pred_esz_masks[5] = {
559	0xffffffffffffffffull, 0x5555555555555555ull,
560	0x1111111111111111ull, 0x0101010101010101ull,
561	0x0001000100010001ull,
562	};
563
564	static bool trans_INVALID(DisasContext s, arg_INVALID a)
565	{
566	unallocated_encoding(s);
567	return true;
568	}
569
570	/*
571	*** SVE Logical - Unpredicated Group
572	*/
573
574	TRANS_FEAT(AND_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_and, a)
575	TRANS_FEAT(ORR_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_or, a)
576	TRANS_FEAT(EOR_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_xor, a)
577	TRANS_FEAT(BIC_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_andc, a)
578
579	static bool trans_XAR(DisasContext s, arg_rrri_esz a)
580	{
581	if (a->esz < 0 \|\| !dc_isar_feature(aa64_sme_or_sve2, s)) {
582	return false;
583	}
584	if (sve_access_check(s)) {
585	unsigned vsz = vec_full_reg_size(s);
586	gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd),
587	vec_full_reg_offset(s, a->rn),
588	vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz);
589	}
590	return true;
591	}
592
593	TRANS_FEAT(EOR3, aa64_sme_or_sve2, gen_gvec_fn_arg_zzzz, gen_gvec_eor3, a)
594	TRANS_FEAT(BCAX, aa64_sme_or_sve2, gen_gvec_fn_arg_zzzz, gen_gvec_bcax, a)
595
596	static void gen_bsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
597	uint32_t a, uint32_t oprsz, uint32_t maxsz)
598	{
599	/* BSL differs from the generic bitsel in argument ordering. */
600	tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz);
601	}
602
603	TRANS_FEAT(BSL, aa64_sme_or_sve2, gen_gvec_fn_arg_zzzz, gen_bsl, a)
604
605	static void gen_bsl1n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
606	{
607	tcg_gen_andc_i64(n, k, n);
608	tcg_gen_andc_i64(m, m, k);
609	tcg_gen_or_i64(d, n, m);
610	}
611
612	static void gen_bsl1n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
613	TCGv_vec m, TCGv_vec k)
614	{
615	tcg_gen_not_vec(vece, n, n);
616	tcg_gen_bitsel_vec(vece, d, k, n, m);
617	}
618
619	static void gen_bsl1n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
620	uint32_t a, uint32_t oprsz, uint32_t maxsz)
621	{
622	static const GVecGen4 op = {
623	.fni8 = gen_bsl1n_i64,
624	.fniv = gen_bsl1n_vec,
625	.fno = gen_helper_sve2_bsl1n,
626	.vece = MO_64,
627	.prefer_i64 = true,
628	};
629	tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
630	}
631
632	TRANS_FEAT(BSL1N, aa64_sme_or_sve2, gen_gvec_fn_arg_zzzz, gen_bsl1n, a)
633
634	static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
635	{
636	/*
637	* Z[dn] = (n & k) \| (~m & ~k)
638	* = \| ~(m \| k)
639	*/
640	tcg_gen_and_i64(n, n, k);
641	if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) {
642	tcg_gen_or_i64(m, m, k);
643	tcg_gen_orc_i64(d, n, m);
644	} else {
645	tcg_gen_nor_i64(m, m, k);
646	tcg_gen_or_i64(d, n, m);
647	}
648	}
649
650	static void gen_bsl2n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
651	TCGv_vec m, TCGv_vec k)
652	{
653	tcg_gen_not_vec(vece, m, m);
654	tcg_gen_bitsel_vec(vece, d, k, n, m);
655	}
656
657	static void gen_bsl2n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
658	uint32_t a, uint32_t oprsz, uint32_t maxsz)
659	{
660	static const GVecGen4 op = {
661	.fni8 = gen_bsl2n_i64,
662	.fniv = gen_bsl2n_vec,
663	.fno = gen_helper_sve2_bsl2n,
664	.vece = MO_64,
665	.prefer_i64 = true,
666	};
667	tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
668	}
669
670	TRANS_FEAT(BSL2N, aa64_sme_or_sve2, gen_gvec_fn_arg_zzzz, gen_bsl2n, a)
671
672	static void gen_nbsl_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
673	{
674	tcg_gen_and_i64(n, n, k);
675	tcg_gen_andc_i64(m, m, k);
676	tcg_gen_nor_i64(d, n, m);
677	}
678
679	static void gen_nbsl_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
680	TCGv_vec m, TCGv_vec k)
681	{
682	tcg_gen_bitsel_vec(vece, d, k, n, m);
683	tcg_gen_not_vec(vece, d, d);
684	}
685
686	static void gen_nbsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
687	uint32_t a, uint32_t oprsz, uint32_t maxsz)
688	{
689	static const GVecGen4 op = {
690	.fni8 = gen_nbsl_i64,
691	.fniv = gen_nbsl_vec,
692	.fno = gen_helper_sve2_nbsl,
693	.vece = MO_64,
694	.prefer_i64 = true,
695	};
696	tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
697	}
698
699	TRANS_FEAT(NBSL, aa64_sme_or_sve2, gen_gvec_fn_arg_zzzz, gen_nbsl, a)
700
701	/*
702	*** SVE Integer Arithmetic - Unpredicated Group
703	*/
704
705	TRANS_FEAT(ADD_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_add, a)
706	TRANS_FEAT(SUB_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sub, a)
707	TRANS_FEAT(SQADD_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ssadd, a)
708	TRANS_FEAT(SQSUB_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sssub, a)
709	TRANS_FEAT(UQADD_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_usadd, a)
710	TRANS_FEAT(UQSUB_zzz, aa64_sme_or_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ussub, a)
711
712	/*
713	*** SVE Integer Arithmetic - Binary Predicated Group
714	*/
715
716	/* Select active elememnts from Zn and inactive elements from Zm,
717	* storing the result in Zd.
718	*/
719	static bool do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz)
720	{
721	static gen_helper_gvec_4 * const fns[4] = {
722	gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
723	gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d
724	};
725	return gen_gvec_ool_zzzp(s, fns[esz], rd, rn, rm, pg, 0);
726	}
727
728	#define DO_ZPZZ(NAME, FEAT, name) \
729	static gen_helper_gvec_4 * const name##_zpzz_fns[4] = { \
730	gen_helper_##name##_zpzz_b, gen_helper_##name##_zpzz_h, \
731	gen_helper_##name##_zpzz_s, gen_helper_##name##_zpzz_d, \
732	}; \
733	TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpzz, \
734	name##_zpzz_fns[a->esz], a, 0)
735
736	DO_ZPZZ(AND_zpzz, aa64_sme_or_sve, sve_and)
737	DO_ZPZZ(EOR_zpzz, aa64_sme_or_sve, sve_eor)
738	DO_ZPZZ(ORR_zpzz, aa64_sme_or_sve, sve_orr)
739	DO_ZPZZ(BIC_zpzz, aa64_sme_or_sve, sve_bic)
740
741	DO_ZPZZ(ADD_zpzz, aa64_sme_or_sve, sve_add)
742	DO_ZPZZ(SUB_zpzz, aa64_sme_or_sve, sve_sub)
743
744	DO_ZPZZ(SMAX_zpzz, aa64_sme_or_sve, sve_smax)
745	DO_ZPZZ(UMAX_zpzz, aa64_sme_or_sve, sve_umax)
746	DO_ZPZZ(SMIN_zpzz, aa64_sme_or_sve, sve_smin)
747	DO_ZPZZ(UMIN_zpzz, aa64_sme_or_sve, sve_umin)
748	DO_ZPZZ(SABD_zpzz, aa64_sme_or_sve, sve_sabd)
749	DO_ZPZZ(UABD_zpzz, aa64_sme_or_sve, sve_uabd)
750
751	DO_ZPZZ(MUL_zpzz, aa64_sme_or_sve, sve_mul)
752	DO_ZPZZ(SMULH_zpzz, aa64_sme_or_sve, sve_smulh)
753	DO_ZPZZ(UMULH_zpzz, aa64_sme_or_sve, sve_umulh)
754
755	DO_ZPZZ(ASR_zpzz, aa64_sme_or_sve, sve_asr)
756	DO_ZPZZ(LSR_zpzz, aa64_sme_or_sve, sve_lsr)
757	DO_ZPZZ(LSL_zpzz, aa64_sme_or_sve, sve_lsl)
758
759	static gen_helper_gvec_4 * const sdiv_fns[4] = {
760	NULL, NULL, gen_helper_sve_sdiv_zpzz_s, gen_helper_sve_sdiv_zpzz_d
761	};
762	TRANS_FEAT(SDIV_zpzz, aa64_sme_or_sve, gen_gvec_ool_arg_zpzz, sdiv_fns[a->esz], a, 0)
763
764	static gen_helper_gvec_4 * const udiv_fns[4] = {
765	NULL, NULL, gen_helper_sve_udiv_zpzz_s, gen_helper_sve_udiv_zpzz_d
766	};
767	TRANS_FEAT(UDIV_zpzz, aa64_sme_or_sve, gen_gvec_ool_arg_zpzz, udiv_fns[a->esz], a, 0)
768
769	TRANS_FEAT(SEL_zpzz, aa64_sme_or_sve, do_sel_z, a->rd, a->rn, a->rm, a->pg, a->esz)
770
771	/*
772	*** SVE Integer Arithmetic - Unary Predicated Group
773	*/
774
775	static gen_helper_gvec_3 * const sve_cls_fns[4] = {
776	gen_helper_sve_cls_b, gen_helper_sve_cls_h,
777	gen_helper_sve_cls_s, gen_helper_sve_cls_d,
778	};
779	TRANS_FEAT(CLS_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz, sve_cls_fns[a->esz], a, 0)
780	TRANS_FEAT(CLS_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz, sve_cls_fns[a->esz], a, 1)
781
782	static gen_helper_gvec_3 * const sve_clz_fns[4] = {
783	gen_helper_sve_clz_b, gen_helper_sve_clz_h,
784	gen_helper_sve_clz_s, gen_helper_sve_clz_d,
785	};
786	TRANS_FEAT(CLZ_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz, sve_clz_fns[a->esz], a, 0)
787	TRANS_FEAT(CLZ_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz, sve_clz_fns[a->esz], a, 1)
788
789	static gen_helper_gvec_3 * const sve_cnt_zpz_fns[4] = {
790	gen_helper_sve_cnt_zpz_b, gen_helper_sve_cnt_zpz_h,
791	gen_helper_sve_cnt_zpz_s, gen_helper_sve_cnt_zpz_d,
792	};
793	TRANS_FEAT(CNT_zpz_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
794	sve_cnt_zpz_fns[a->esz], a, 0)
795	TRANS_FEAT(CNT_zpz_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
796	sve_cnt_zpz_fns[a->esz], a, 1)
797
798	static gen_helper_gvec_3 * const sve_cnot_fns[4] = {
799	gen_helper_sve_cnot_b, gen_helper_sve_cnot_h,
800	gen_helper_sve_cnot_s, gen_helper_sve_cnot_d,
801	};
802	TRANS_FEAT(CNOT_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
803	sve_cnot_fns[a->esz], a, 0)
804	TRANS_FEAT(CNOT_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
805	sve_cnot_fns[a->esz], a, 1)
806
807	static gen_helper_gvec_3 * const sve_not_zpz_fns[4] = {
808	gen_helper_sve_not_zpz_b, gen_helper_sve_not_zpz_h,
809	gen_helper_sve_not_zpz_s, gen_helper_sve_not_zpz_d,
810	};
811	TRANS_FEAT(NOT_zpz_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
812	sve_not_zpz_fns[a->esz], a, 0)
813	TRANS_FEAT(NOT_zpz_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
814	sve_not_zpz_fns[a->esz], a, 1)
815
816	static gen_helper_gvec_3 * const sve_abs_fns[4] = {
817	gen_helper_sve_abs_b, gen_helper_sve_abs_h,
818	gen_helper_sve_abs_s, gen_helper_sve_abs_d,
819	};
820	TRANS_FEAT(ABS_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz, sve_abs_fns[a->esz], a, 0)
821	TRANS_FEAT(ABS_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz, sve_abs_fns[a->esz], a, 1)
822
823	static gen_helper_gvec_3 * const sve_neg_fns[4] = {
824	gen_helper_sve_neg_b, gen_helper_sve_neg_h,
825	gen_helper_sve_neg_s, gen_helper_sve_neg_d,
826	};
827	TRANS_FEAT(NEG_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz, sve_neg_fns[a->esz], a, 0)
828	TRANS_FEAT(NEG_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz, sve_neg_fns[a->esz], a, 1)
829
830	static gen_helper_gvec_3 * const sve_rbit_fns[4] = {
831	gen_helper_sve_rbit_b, gen_helper_sve_rbit_h,
832	gen_helper_sve_rbit_s, gen_helper_sve_rbit_d,
833	};
834	TRANS_FEAT(RBIT_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz, sve_rbit_fns[a->esz], a, 0)
835	TRANS_FEAT(RBIT_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz, sve_rbit_fns[a->esz], a, 1)
836
837	static gen_helper_gvec_3 * const sve2p1_orqv_fns[4] = {
838	gen_helper_sve2p1_orqv_b, gen_helper_sve2p1_orqv_h,
839	gen_helper_sve2p1_orqv_s, gen_helper_sve2p1_orqv_d,
840	};
841	TRANS_FEAT(ORQV, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zpz,
842	sve2p1_orqv_fns[a->esz], a, 0)
843
844	static gen_helper_gvec_3 * const sve2p1_eorqv_fns[4] = {
845	gen_helper_sve2p1_eorqv_b, gen_helper_sve2p1_eorqv_h,
846	gen_helper_sve2p1_eorqv_s, gen_helper_sve2p1_eorqv_d,
847	};
848	TRANS_FEAT(EORQV, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zpz,
849	sve2p1_eorqv_fns[a->esz], a, 0)
850
851	static gen_helper_gvec_3 * const sve2p1_andqv_fns[4] = {
852	gen_helper_sve2p1_andqv_b, gen_helper_sve2p1_andqv_h,
853	gen_helper_sve2p1_andqv_s, gen_helper_sve2p1_andqv_d,
854	};
855	TRANS_FEAT(ANDQV, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zpz,
856	sve2p1_andqv_fns[a->esz], a, 0)
857
858	static gen_helper_gvec_3 * const fabs_fns[4] = {
859	NULL, gen_helper_sve_fabs_h,
860	gen_helper_sve_fabs_s, gen_helper_sve_fabs_d,
861	};
862	static gen_helper_gvec_3 * const fabs_ah_fns[4] = {
863	NULL, gen_helper_sve_ah_fabs_h,
864	gen_helper_sve_ah_fabs_s, gen_helper_sve_ah_fabs_d,
865	};
866	TRANS_FEAT(FABS_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
867	s->fpcr_ah ? fabs_ah_fns[a->esz] : fabs_fns[a->esz], a, 0)
868	TRANS_FEAT(FABS_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
869	s->fpcr_ah ? fabs_ah_fns[a->esz] : fabs_fns[a->esz], a, 1)
870
871	static gen_helper_gvec_3 * const fneg_fns[4] = {
872	NULL, gen_helper_sve_fneg_h,
873	gen_helper_sve_fneg_s, gen_helper_sve_fneg_d,
874	};
875	static gen_helper_gvec_3 * const fneg_ah_fns[4] = {
876	NULL, gen_helper_sve_ah_fneg_h,
877	gen_helper_sve_ah_fneg_s, gen_helper_sve_ah_fneg_d,
878	};
879	TRANS_FEAT(FNEG_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
880	s->fpcr_ah ? fneg_ah_fns[a->esz] : fneg_fns[a->esz], a, 0)
881	TRANS_FEAT(FNEG_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
882	s->fpcr_ah ? fneg_ah_fns[a->esz] : fneg_fns[a->esz], a, 1)
883
884	static gen_helper_gvec_3 * const sxtb_fns[4] = {
885	NULL, gen_helper_sve_sxtb_h,
886	gen_helper_sve_sxtb_s, gen_helper_sve_sxtb_d,
887	};
888	TRANS_FEAT(SXTB_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
889	sxtb_fns[a->esz], a, 0)
890	TRANS_FEAT(SXTB_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
891	sxtb_fns[a->esz], a, 1)
892
893	static gen_helper_gvec_3 * const uxtb_fns[4] = {
894	NULL, gen_helper_sve_uxtb_h,
895	gen_helper_sve_uxtb_s, gen_helper_sve_uxtb_d,
896	};
897	TRANS_FEAT(UXTB_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
898	uxtb_fns[a->esz], a, 0)
899	TRANS_FEAT(UXTB_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
900	uxtb_fns[a->esz], a, 1)
901
902	static gen_helper_gvec_3 * const sxth_fns[4] = {
903	NULL, NULL, gen_helper_sve_sxth_s, gen_helper_sve_sxth_d
904	};
905	TRANS_FEAT(SXTH_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
906	sxth_fns[a->esz], a, 0)
907	TRANS_FEAT(SXTH_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
908	sxth_fns[a->esz], a, 1)
909
910	static gen_helper_gvec_3 * const uxth_fns[4] = {
911	NULL, NULL, gen_helper_sve_uxth_s, gen_helper_sve_uxth_d
912	};
913	TRANS_FEAT(UXTH_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
914	uxth_fns[a->esz], a, 0)
915	TRANS_FEAT(UXTH_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
916	uxth_fns[a->esz], a, 1)
917
918	TRANS_FEAT(SXTW_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
919	a->esz == 3 ? gen_helper_sve_sxtw_d : NULL, a, 0)
920	TRANS_FEAT(SXTW_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
921	a->esz == 3 ? gen_helper_sve_sxtw_d : NULL, a, 1)
922
923	TRANS_FEAT(UXTW_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
924	a->esz == 3 ? gen_helper_sve_uxtw_d : NULL, a, 0)
925	TRANS_FEAT(UXTW_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
926	a->esz == 3 ? gen_helper_sve_uxtw_d : NULL, a, 1)
927
928	static gen_helper_gvec_3 * const addqv_fns[4] = {
929	gen_helper_sve2p1_addqv_b, gen_helper_sve2p1_addqv_h,
930	gen_helper_sve2p1_addqv_s, gen_helper_sve2p1_addqv_d,
931	};
932	TRANS_FEAT(ADDQV, aa64_sme2p1_or_sve2p1,
933	gen_gvec_ool_arg_zpz, addqv_fns[a->esz], a, 0)
934
935	static gen_helper_gvec_3 * const smaxqv_fns[4] = {
936	gen_helper_sve2p1_smaxqv_b, gen_helper_sve2p1_smaxqv_h,
937	gen_helper_sve2p1_smaxqv_s, gen_helper_sve2p1_smaxqv_d,
938	};
939	TRANS_FEAT(SMAXQV, aa64_sme2p1_or_sve2p1,
940	gen_gvec_ool_arg_zpz, smaxqv_fns[a->esz], a, 0)
941
942	static gen_helper_gvec_3 * const sminqv_fns[4] = {
943	gen_helper_sve2p1_sminqv_b, gen_helper_sve2p1_sminqv_h,
944	gen_helper_sve2p1_sminqv_s, gen_helper_sve2p1_sminqv_d,
945	};
946	TRANS_FEAT(SMINQV, aa64_sme2p1_or_sve2p1,
947	gen_gvec_ool_arg_zpz, sminqv_fns[a->esz], a, 0)
948
949	static gen_helper_gvec_3 * const umaxqv_fns[4] = {
950	gen_helper_sve2p1_umaxqv_b, gen_helper_sve2p1_umaxqv_h,
951	gen_helper_sve2p1_umaxqv_s, gen_helper_sve2p1_umaxqv_d,
952	};
953	TRANS_FEAT(UMAXQV, aa64_sme2p1_or_sve2p1,
954	gen_gvec_ool_arg_zpz, umaxqv_fns[a->esz], a, 0)
955
956	static gen_helper_gvec_3 * const uminqv_fns[4] = {
957	gen_helper_sve2p1_uminqv_b, gen_helper_sve2p1_uminqv_h,
958	gen_helper_sve2p1_uminqv_s, gen_helper_sve2p1_uminqv_d,
959	};
960	TRANS_FEAT(UMINQV, aa64_sme2p1_or_sve2p1,
961	gen_gvec_ool_arg_zpz, uminqv_fns[a->esz], a, 0)
962
963	/*
964	*** SVE Integer Reduction Group
965	*/
966
967	typedef void gen_helper_gvec_reduc(TCGv_i64, TCGv_ptr, TCGv_ptr, TCGv_i32);
968	static bool do_vpz_ool(DisasContext s, arg_rpr_esz a,
969	gen_helper_gvec_reduc *fn)
970	{
971	unsigned vsz = vec_full_reg_size(s);
972	TCGv_ptr t_zn, t_pg;
973	TCGv_i32 desc;
974	TCGv_i64 temp;
975
976	if (fn == NULL) {
977	return false;
978	}
979	if (!sve_access_check(s)) {
980	return true;
981	}
982
983	desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
984	temp = tcg_temp_new_i64();
985	t_zn = tcg_temp_new_ptr();
986	t_pg = tcg_temp_new_ptr();
987
988	tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn));
989	tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg));
990	fn(temp, t_zn, t_pg, desc);
991
992	write_fp_dreg(s, a->rd, temp);
993	return true;
994	}
995
996	#define DO_VPZ(NAME, name) \
997	static gen_helper_gvec_reduc * const name##_fns[4] = { \
998	gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
999	gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1000	}; \
1001	TRANS_FEAT(NAME, aa64_sme_or_sve, do_vpz_ool, a, name##_fns[a->esz])
1002
1003	DO_VPZ(ORV, orv)
1004	DO_VPZ(ANDV, andv)
1005	DO_VPZ(EORV, eorv)
1006
1007	DO_VPZ(UADDV, uaddv)
1008	DO_VPZ(SMAXV, smaxv)
1009	DO_VPZ(UMAXV, umaxv)
1010	DO_VPZ(SMINV, sminv)
1011	DO_VPZ(UMINV, uminv)
1012
1013	static gen_helper_gvec_reduc * const saddv_fns[4] = {
1014	gen_helper_sve_saddv_b, gen_helper_sve_saddv_h,
1015	gen_helper_sve_saddv_s, NULL
1016	};
1017	TRANS_FEAT(SADDV, aa64_sme_or_sve, do_vpz_ool, a, saddv_fns[a->esz])
1018
1019	#undef DO_VPZ
1020
1021	/*
1022	*** SVE Shift by Immediate - Predicated Group
1023	*/
1024
1025	/*
1026	* Copy Zn into Zd, storing zeros into inactive elements.
1027	* If invert, store zeros into the active elements.
1028	*/
1029	static bool do_movz_zpz(DisasContext *s, int rd, int rn, int pg,
1030	int esz, bool invert)
1031	{
1032	static gen_helper_gvec_3 * const fns[4] = {
1033	gen_helper_sve_movz_b, gen_helper_sve_movz_h,
1034	gen_helper_sve_movz_s, gen_helper_sve_movz_d,
1035	};
1036	return gen_gvec_ool_zzp(s, fns[esz], rd, rn, pg, invert);
1037	}
1038
1039	static bool do_shift_zpzi(DisasContext s, arg_rpri_esz a, bool asr,
1040	gen_helper_gvec_3 * const fns[4])
1041	{
1042	int max;
1043
1044	if (a->esz < 0) {
1045	/* Invalid tsz encoding -- see tszimm_esz. */
1046	return false;
1047	}
1048
1049	/*
1050	* Shift by element size is architecturally valid.
1051	* For arithmetic right-shift, it's the same as by one less.
1052	* For logical shifts and ASRD, it is a zeroing operation.
1053	*/
1054	max = 8 << a->esz;
1055	if (a->imm >= max) {
1056	if (asr) {
1057	a->imm = max - 1;
1058	} else {
1059	return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true);
1060	}
1061	}
1062	return gen_gvec_ool_arg_zpzi(s, fns[a->esz], a);
1063	}
1064
1065	static gen_helper_gvec_3 * const asr_zpzi_fns[4] = {
1066	gen_helper_sve_asr_zpzi_b, gen_helper_sve_asr_zpzi_h,
1067	gen_helper_sve_asr_zpzi_s, gen_helper_sve_asr_zpzi_d,
1068	};
1069	TRANS_FEAT(ASR_zpzi, aa64_sme_or_sve, do_shift_zpzi, a, true, asr_zpzi_fns)
1070
1071	static gen_helper_gvec_3 * const lsr_zpzi_fns[4] = {
1072	gen_helper_sve_lsr_zpzi_b, gen_helper_sve_lsr_zpzi_h,
1073	gen_helper_sve_lsr_zpzi_s, gen_helper_sve_lsr_zpzi_d,
1074	};
1075	TRANS_FEAT(LSR_zpzi, aa64_sme_or_sve, do_shift_zpzi, a, false, lsr_zpzi_fns)
1076
1077	static gen_helper_gvec_3 * const lsl_zpzi_fns[4] = {
1078	gen_helper_sve_lsl_zpzi_b, gen_helper_sve_lsl_zpzi_h,
1079	gen_helper_sve_lsl_zpzi_s, gen_helper_sve_lsl_zpzi_d,
1080	};
1081	TRANS_FEAT(LSL_zpzi, aa64_sme_or_sve, do_shift_zpzi, a, false, lsl_zpzi_fns)
1082
1083	static gen_helper_gvec_3 * const asrd_fns[4] = {
1084	gen_helper_sve_asrd_b, gen_helper_sve_asrd_h,
1085	gen_helper_sve_asrd_s, gen_helper_sve_asrd_d,
1086	};
1087	TRANS_FEAT(ASRD, aa64_sme_or_sve, do_shift_zpzi, a, false, asrd_fns)
1088
1089	static gen_helper_gvec_3 * const sqshl_zpzi_fns[4] = {
1090	gen_helper_sve2_sqshl_zpzi_b, gen_helper_sve2_sqshl_zpzi_h,
1091	gen_helper_sve2_sqshl_zpzi_s, gen_helper_sve2_sqshl_zpzi_d,
1092	};
1093	TRANS_FEAT(SQSHL_zpzi, aa64_sme_or_sve2, gen_gvec_ool_arg_zpzi,
1094	a->esz < 0 ? NULL : sqshl_zpzi_fns[a->esz], a)
1095
1096	static gen_helper_gvec_3 * const uqshl_zpzi_fns[4] = {
1097	gen_helper_sve2_uqshl_zpzi_b, gen_helper_sve2_uqshl_zpzi_h,
1098	gen_helper_sve2_uqshl_zpzi_s, gen_helper_sve2_uqshl_zpzi_d,
1099	};
1100	TRANS_FEAT(UQSHL_zpzi, aa64_sme_or_sve2, gen_gvec_ool_arg_zpzi,
1101	a->esz < 0 ? NULL : uqshl_zpzi_fns[a->esz], a)
1102
1103	static gen_helper_gvec_3 * const srshr_fns[4] = {
1104	gen_helper_sve2_srshr_b, gen_helper_sve2_srshr_h,
1105	gen_helper_sve2_srshr_s, gen_helper_sve2_srshr_d,
1106	};
1107	TRANS_FEAT(SRSHR, aa64_sme_or_sve2, gen_gvec_ool_arg_zpzi,
1108	a->esz < 0 ? NULL : srshr_fns[a->esz], a)
1109
1110	static gen_helper_gvec_3 * const urshr_fns[4] = {
1111	gen_helper_sve2_urshr_b, gen_helper_sve2_urshr_h,
1112	gen_helper_sve2_urshr_s, gen_helper_sve2_urshr_d,
1113	};
1114	TRANS_FEAT(URSHR, aa64_sme_or_sve2, gen_gvec_ool_arg_zpzi,
1115	a->esz < 0 ? NULL : urshr_fns[a->esz], a)
1116
1117	static gen_helper_gvec_3 * const sqshlu_fns[4] = {
1118	gen_helper_sve2_sqshlu_b, gen_helper_sve2_sqshlu_h,
1119	gen_helper_sve2_sqshlu_s, gen_helper_sve2_sqshlu_d,
1120	};
1121	TRANS_FEAT(SQSHLU, aa64_sme_or_sve2, gen_gvec_ool_arg_zpzi,
1122	a->esz < 0 ? NULL : sqshlu_fns[a->esz], a)
1123
1124	/*
1125	*** SVE Bitwise Shift - Predicated Group
1126	*/
1127
1128	#define DO_ZPZW(NAME, name) \
1129	static gen_helper_gvec_4 * const name##_zpzw_fns[4] = { \
1130	gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1131	gen_helper_sve_##name##_zpzw_s, NULL \
1132	}; \
1133	TRANS_FEAT(NAME##_zpzw, aa64_sme_or_sve, gen_gvec_ool_arg_zpzz, \
1134	a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0)
1135
1136	DO_ZPZW(ASR, asr)
1137	DO_ZPZW(LSR, lsr)
1138	DO_ZPZW(LSL, lsl)
1139
1140	#undef DO_ZPZW
1141
1142	/*
1143	*** SVE Bitwise Shift - Unpredicated Group
1144	*/
1145
1146	static bool do_shift_imm(DisasContext s, arg_rri_esz a, bool asr,
1147	void (*gvec_fn)(unsigned, uint32_t, uint32_t,
1148	int64_t, uint32_t, uint32_t))
1149	{
1150	if (a->esz < 0) {
1151	/* Invalid tsz encoding -- see tszimm_esz. */
1152	return false;
1153	}
1154	if (sve_access_check(s)) {
1155	unsigned vsz = vec_full_reg_size(s);
1156	/* Shift by element size is architecturally valid. For
1157	arithmetic right-shift, it's the same as by one less.
1158	Otherwise it is a zeroing operation. */
1159	if (a->imm >= 8 << a->esz) {
1160	if (asr) {
1161	a->imm = (8 << a->esz) - 1;
1162	} else {
1163	do_dupi_z(s, a->rd, 0);
1164	return true;
1165	}
1166	}
1167	gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
1168	vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
1169	}
1170	return true;
1171	}
1172
1173	TRANS_FEAT(ASR_zzi, aa64_sme_or_sve, do_shift_imm, a, true, tcg_gen_gvec_sari)
1174	TRANS_FEAT(LSR_zzi, aa64_sme_or_sve, do_shift_imm, a, false, tcg_gen_gvec_shri)
1175	TRANS_FEAT(LSL_zzi, aa64_sme_or_sve, do_shift_imm, a, false, tcg_gen_gvec_shli)
1176
1177	#define DO_ZZW(NAME, name) \
1178	static gen_helper_gvec_3 * const name##_zzw_fns[4] = { \
1179	gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1180	gen_helper_sve_##name##_zzw_s, NULL \
1181	}; \
1182	TRANS_FEAT(NAME, aa64_sme_or_sve, gen_gvec_ool_arg_zzz, \
1183	name##_zzw_fns[a->esz], a, 0)
1184
1185	DO_ZZW(ASR_zzw, asr)
1186	DO_ZZW(LSR_zzw, lsr)
1187	DO_ZZW(LSL_zzw, lsl)
1188
1189	#undef DO_ZZW
1190
1191	/*
1192	*** SVE Integer Multiply-Add Group
1193	*/
1194
1195	static bool do_zpzzz_ool(DisasContext s, arg_rprrr_esz a,
1196	gen_helper_gvec_5 *fn)
1197	{
1198	if (sve_access_check(s)) {
1199	unsigned vsz = vec_full_reg_size(s);
1200	tcg_gen_gvec_5_ool(vec_full_reg_offset(s, a->rd),
1201	vec_full_reg_offset(s, a->ra),
1202	vec_full_reg_offset(s, a->rn),
1203	vec_full_reg_offset(s, a->rm),
1204	pred_full_reg_offset(s, a->pg),
1205	vsz, vsz, 0, fn);
1206	}
1207	return true;
1208	}
1209
1210	static gen_helper_gvec_5 * const mla_fns[4] = {
1211	gen_helper_sve_mla_b, gen_helper_sve_mla_h,
1212	gen_helper_sve_mla_s, gen_helper_sve_mla_d,
1213	};
1214	TRANS_FEAT(MLA, aa64_sme_or_sve, do_zpzzz_ool, a, mla_fns[a->esz])
1215
1216	static gen_helper_gvec_5 * const mls_fns[4] = {
1217	gen_helper_sve_mls_b, gen_helper_sve_mls_h,
1218	gen_helper_sve_mls_s, gen_helper_sve_mls_d,
1219	};
1220	TRANS_FEAT(MLS, aa64_sme_or_sve, do_zpzzz_ool, a, mls_fns[a->esz])
1221
1222	/*
1223	*** SVE Index Generation Group
1224	*/
1225
1226	static bool do_index(DisasContext *s, int esz, int rd,
1227	TCGv_i64 start, TCGv_i64 incr)
1228	{
1229	unsigned vsz;
1230	TCGv_i32 desc;
1231	TCGv_ptr t_zd;
1232
1233	if (!sve_access_check(s)) {
1234	return true;
1235	}
1236
1237	vsz = vec_full_reg_size(s);
1238	desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
1239	t_zd = tcg_temp_new_ptr();
1240
1241	tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, rd));
1242	if (esz == 3) {
1243	gen_helper_sve_index_d(t_zd, start, incr, desc);
1244	} else {
1245	typedef void index_fn(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32);
1246	static index_fn * const fns[3] = {
1247	gen_helper_sve_index_b,
1248	gen_helper_sve_index_h,
1249	gen_helper_sve_index_s,
1250	};
1251	TCGv_i32 s32 = tcg_temp_new_i32();
1252	TCGv_i32 i32 = tcg_temp_new_i32();
1253
1254	tcg_gen_extrl_i64_i32(s32, start);
1255	tcg_gen_extrl_i64_i32(i32, incr);
1256	fns[esz](t_zd, s32, i32, desc);
1257	}
1258	return true;
1259	}
1260
1261	TRANS_FEAT(INDEX_ii, aa64_sme_or_sve, do_index, a->esz, a->rd,
1262	tcg_constant_i64(a->imm1), tcg_constant_i64(a->imm2))
1263	TRANS_FEAT(INDEX_ir, aa64_sme_or_sve, do_index, a->esz, a->rd,
1264	tcg_constant_i64(a->imm), cpu_reg(s, a->rm))
1265	TRANS_FEAT(INDEX_ri, aa64_sme_or_sve, do_index, a->esz, a->rd,
1266	cpu_reg(s, a->rn), tcg_constant_i64(a->imm))
1267	TRANS_FEAT(INDEX_rr, aa64_sme_or_sve, do_index, a->esz, a->rd,
1268	cpu_reg(s, a->rn), cpu_reg(s, a->rm))
1269
1270	/*
1271	*** SVE Stack Allocation Group
1272	*/
1273
1274	static bool trans_ADDVL(DisasContext s, arg_ADDVL a)
1275	{
1276	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1277	return false;
1278	}
1279	if (sve_access_check(s)) {
1280	TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1281	TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1282	tcg_gen_addi_i64(rd, rn, a->imm * vec_full_reg_size(s));
1283	}
1284	return true;
1285	}
1286
1287	static bool trans_ADDSVL(DisasContext s, arg_ADDSVL a)
1288	{
1289	if (!dc_isar_feature(aa64_sme, s)) {
1290	return false;
1291	}
1292	if (sme_enabled_check(s)) {
1293	TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1294	TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1295	tcg_gen_addi_i64(rd, rn, a->imm * streaming_vec_reg_size(s));
1296	}
1297	return true;
1298	}
1299
1300	static bool trans_ADDPL(DisasContext s, arg_ADDPL a)
1301	{
1302	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1303	return false;
1304	}
1305	if (sve_access_check(s)) {
1306	TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1307	TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1308	tcg_gen_addi_i64(rd, rn, a->imm * pred_full_reg_size(s));
1309	}
1310	return true;
1311	}
1312
1313	static bool trans_ADDSPL(DisasContext s, arg_ADDSPL a)
1314	{
1315	if (!dc_isar_feature(aa64_sme, s)) {
1316	return false;
1317	}
1318	if (sme_enabled_check(s)) {
1319	TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1320	TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1321	tcg_gen_addi_i64(rd, rn, a->imm * streaming_pred_reg_size(s));
1322	}
1323	return true;
1324	}
1325
1326	static bool trans_RDVL(DisasContext s, arg_RDVL a)
1327	{
1328	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1329	return false;
1330	}
1331	if (sve_access_check(s)) {
1332	TCGv_i64 reg = cpu_reg(s, a->rd);
1333	tcg_gen_movi_i64(reg, a->imm * vec_full_reg_size(s));
1334	}
1335	return true;
1336	}
1337
1338	static bool trans_RDSVL(DisasContext s, arg_RDSVL a)
1339	{
1340	if (!dc_isar_feature(aa64_sme, s)) {
1341	return false;
1342	}
1343	if (sme_enabled_check(s)) {
1344	TCGv_i64 reg = cpu_reg(s, a->rd);
1345	tcg_gen_movi_i64(reg, a->imm * streaming_vec_reg_size(s));
1346	}
1347	return true;
1348	}
1349
1350	/*
1351	*** SVE Compute Vector Address Group
1352	*/
1353
1354	static bool do_adr(DisasContext s, arg_rrri a, gen_helper_gvec_3 *fn)
1355	{
1356	return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm);
1357	}
1358
1359	TRANS_FEAT_NONSTREAMING(ADR_p32, aa64_sve, do_adr, a, gen_helper_sve_adr_p32)
1360	TRANS_FEAT_NONSTREAMING(ADR_p64, aa64_sve, do_adr, a, gen_helper_sve_adr_p64)
1361	TRANS_FEAT_NONSTREAMING(ADR_s32, aa64_sve, do_adr, a, gen_helper_sve_adr_s32)
1362	TRANS_FEAT_NONSTREAMING(ADR_u32, aa64_sve, do_adr, a, gen_helper_sve_adr_u32)
1363
1364	/*
1365	*** SVE Integer Misc - Unpredicated Group
1366	*/
1367
1368	static bool trans_FEXPA(DisasContext s, arg_FEXPA a)
1369	{
1370	static gen_helper_gvec_2 * const fexpa_fns[4] = {
1371	NULL, gen_helper_sve_fexpa_h,
1372	gen_helper_sve_fexpa_s, gen_helper_sve_fexpa_d,
1373	};
1374
1375	if (!dc_isar_feature(aa64_ssve_fexpa, s)) {
1376	if (!dc_isar_feature(aa64_sve, s)) {
1377	return false;
1378	}
1379	s->is_nonstreaming = true;
1380	}
1381	return gen_gvec_ool_zz(s, fexpa_fns[a->esz], a->rd, a->rn, s->fpcr_ah);
1382	}
1383
1384	static gen_helper_gvec_3 * const ftssel_fns[4] = {
1385	NULL, gen_helper_sve_ftssel_h,
1386	gen_helper_sve_ftssel_s, gen_helper_sve_ftssel_d,
1387	};
1388	TRANS_FEAT_NONSTREAMING(FTSSEL, aa64_sve, gen_gvec_ool_arg_zzz,
1389	ftssel_fns[a->esz], a, s->fpcr_ah)
1390
1391	/*
1392	*** SVE Predicate Logical Operations Group
1393	*/
1394
1395	static bool do_pppp_flags(DisasContext s, arg_rprr_s a,
1396	const GVecGen4 *gvec_op)
1397	{
1398	if (!sve_access_check(s)) {
1399	return true;
1400	}
1401
1402	unsigned psz = pred_gvec_reg_size(s);
1403	int dofs = pred_full_reg_offset(s, a->rd);
1404	int nofs = pred_full_reg_offset(s, a->rn);
1405	int mofs = pred_full_reg_offset(s, a->rm);
1406	int gofs = pred_full_reg_offset(s, a->pg);
1407
1408	if (!a->s) {
1409	tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op);
1410	return true;
1411	}
1412
1413	if (psz == 8) {
1414	/* Do the operation and the flags generation in temps. */
1415	TCGv_i64 pd = tcg_temp_new_i64();
1416	TCGv_i64 pn = tcg_temp_new_i64();
1417	TCGv_i64 pm = tcg_temp_new_i64();
1418	TCGv_i64 pg = tcg_temp_new_i64();
1419
1420	tcg_gen_ld_i64(pn, tcg_env, nofs);
1421	tcg_gen_ld_i64(pm, tcg_env, mofs);
1422	tcg_gen_ld_i64(pg, tcg_env, gofs);
1423
1424	gvec_op->fni8(pd, pn, pm, pg);
1425	tcg_gen_st_i64(pd, tcg_env, dofs);
1426
1427	do_predtest1(pd, pg);
1428	} else {
1429	/* The operation and flags generation is large. The computation
1430	* of the flags depends on the original contents of the guarding
1431	* predicate. If the destination overwrites the guarding predicate,
1432	* then the easiest way to get this right is to save a copy.
1433	*/
1434	int tofs = gofs;
1435	if (a->rd == a->pg) {
1436	tofs = offsetof(CPUARMState, vfp.preg_tmp);
1437	tcg_gen_gvec_mov(0, tofs, gofs, psz, psz);
1438	}
1439
1440	tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op);
1441	do_predtest(s, dofs, tofs, psz / 8);
1442	}
1443	return true;
1444	}
1445
1446	static void gen_and_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1447	{
1448	tcg_gen_and_i64(pd, pn, pm);
1449	tcg_gen_and_i64(pd, pd, pg);
1450	}
1451
1452	static void gen_and_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1453	TCGv_vec pm, TCGv_vec pg)
1454	{
1455	tcg_gen_and_vec(vece, pd, pn, pm);
1456	tcg_gen_and_vec(vece, pd, pd, pg);
1457	}
1458
1459	static bool trans_AND_pppp(DisasContext s, arg_rprr_s a)
1460	{
1461	static const GVecGen4 op = {
1462	.fni8 = gen_and_pg_i64,
1463	.fniv = gen_and_pg_vec,
1464	.fno = gen_helper_sve_and_pppp,
1465	.prefer_i64 = true,
1466	};
1467
1468	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1469	return false;
1470	}
1471	if (!a->s) {
1472	if (a->rn == a->rm) {
1473	if (a->pg == a->rn) {
1474	return do_mov_p(s, a->rd, a->rn);
1475	}
1476	return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg);
1477	} else if (a->pg == a->rn \|\| a->pg == a->rm) {
1478	return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm);
1479	}
1480	}
1481	return do_pppp_flags(s, a, &op);
1482	}
1483
1484	static void gen_bic_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1485	{
1486	tcg_gen_andc_i64(pd, pn, pm);
1487	tcg_gen_and_i64(pd, pd, pg);
1488	}
1489
1490	static void gen_bic_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1491	TCGv_vec pm, TCGv_vec pg)
1492	{
1493	tcg_gen_andc_vec(vece, pd, pn, pm);
1494	tcg_gen_and_vec(vece, pd, pd, pg);
1495	}
1496
1497	static bool trans_BIC_pppp(DisasContext s, arg_rprr_s a)
1498	{
1499	static const GVecGen4 op = {
1500	.fni8 = gen_bic_pg_i64,
1501	.fniv = gen_bic_pg_vec,
1502	.fno = gen_helper_sve_bic_pppp,
1503	.prefer_i64 = true,
1504	};
1505
1506	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1507	return false;
1508	}
1509	if (!a->s && a->pg == a->rn) {
1510	return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm);
1511	}
1512	return do_pppp_flags(s, a, &op);
1513	}
1514
1515	static void gen_eor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1516	{
1517	tcg_gen_xor_i64(pd, pn, pm);
1518	tcg_gen_and_i64(pd, pd, pg);
1519	}
1520
1521	static void gen_eor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1522	TCGv_vec pm, TCGv_vec pg)
1523	{
1524	tcg_gen_xor_vec(vece, pd, pn, pm);
1525	tcg_gen_and_vec(vece, pd, pd, pg);
1526	}
1527
1528	static bool trans_EOR_pppp(DisasContext s, arg_rprr_s a)
1529	{
1530	static const GVecGen4 op = {
1531	.fni8 = gen_eor_pg_i64,
1532	.fniv = gen_eor_pg_vec,
1533	.fno = gen_helper_sve_eor_pppp,
1534	.prefer_i64 = true,
1535	};
1536
1537	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1538	return false;
1539	}
1540	/* Alias NOT (predicate) is EOR Pd.B, Pg/Z, Pn.B, Pg.B */
1541	if (!a->s && a->pg == a->rm) {
1542	return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->pg, a->rn);
1543	}
1544	return do_pppp_flags(s, a, &op);
1545	}
1546
1547	static bool trans_SEL_pppp(DisasContext s, arg_rprr_s a)
1548	{
1549	if (a->s \|\| !dc_isar_feature(aa64_sme_or_sve, s)) {
1550	return false;
1551	}
1552	if (sve_access_check(s)) {
1553	unsigned psz = pred_gvec_reg_size(s);
1554	tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd),
1555	pred_full_reg_offset(s, a->pg),
1556	pred_full_reg_offset(s, a->rn),
1557	pred_full_reg_offset(s, a->rm), psz, psz);
1558	}
1559	return true;
1560	}
1561
1562	static void gen_orr_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1563	{
1564	tcg_gen_or_i64(pd, pn, pm);
1565	tcg_gen_and_i64(pd, pd, pg);
1566	}
1567
1568	static void gen_orr_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1569	TCGv_vec pm, TCGv_vec pg)
1570	{
1571	tcg_gen_or_vec(vece, pd, pn, pm);
1572	tcg_gen_and_vec(vece, pd, pd, pg);
1573	}
1574
1575	static bool trans_ORR_pppp(DisasContext s, arg_rprr_s a)
1576	{
1577	static const GVecGen4 op = {
1578	.fni8 = gen_orr_pg_i64,
1579	.fniv = gen_orr_pg_vec,
1580	.fno = gen_helper_sve_orr_pppp,
1581	.prefer_i64 = true,
1582	};
1583
1584	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1585	return false;
1586	}
1587	if (!a->s && a->pg == a->rn && a->rn == a->rm) {
1588	return do_mov_p(s, a->rd, a->rn);
1589	}
1590	return do_pppp_flags(s, a, &op);
1591	}
1592
1593	static void gen_orn_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1594	{
1595	tcg_gen_orc_i64(pd, pn, pm);
1596	tcg_gen_and_i64(pd, pd, pg);
1597	}
1598
1599	static void gen_orn_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1600	TCGv_vec pm, TCGv_vec pg)
1601	{
1602	tcg_gen_orc_vec(vece, pd, pn, pm);
1603	tcg_gen_and_vec(vece, pd, pd, pg);
1604	}
1605
1606	static bool trans_ORN_pppp(DisasContext s, arg_rprr_s a)
1607	{
1608	static const GVecGen4 op = {
1609	.fni8 = gen_orn_pg_i64,
1610	.fniv = gen_orn_pg_vec,
1611	.fno = gen_helper_sve_orn_pppp,
1612	.prefer_i64 = true,
1613	};
1614
1615	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1616	return false;
1617	}
1618	return do_pppp_flags(s, a, &op);
1619	}
1620
1621	static void gen_nor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1622	{
1623	tcg_gen_or_i64(pd, pn, pm);
1624	tcg_gen_andc_i64(pd, pg, pd);
1625	}
1626
1627	static void gen_nor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1628	TCGv_vec pm, TCGv_vec pg)
1629	{
1630	tcg_gen_or_vec(vece, pd, pn, pm);
1631	tcg_gen_andc_vec(vece, pd, pg, pd);
1632	}
1633
1634	static bool trans_NOR_pppp(DisasContext s, arg_rprr_s a)
1635	{
1636	static const GVecGen4 op = {
1637	.fni8 = gen_nor_pg_i64,
1638	.fniv = gen_nor_pg_vec,
1639	.fno = gen_helper_sve_nor_pppp,
1640	.prefer_i64 = true,
1641	};
1642
1643	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1644	return false;
1645	}
1646	return do_pppp_flags(s, a, &op);
1647	}
1648
1649	static void gen_nand_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1650	{
1651	tcg_gen_and_i64(pd, pn, pm);
1652	tcg_gen_andc_i64(pd, pg, pd);
1653	}
1654
1655	static void gen_nand_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1656	TCGv_vec pm, TCGv_vec pg)
1657	{
1658	tcg_gen_and_vec(vece, pd, pn, pm);
1659	tcg_gen_andc_vec(vece, pd, pg, pd);
1660	}
1661
1662	static bool trans_NAND_pppp(DisasContext s, arg_rprr_s a)
1663	{
1664	static const GVecGen4 op = {
1665	.fni8 = gen_nand_pg_i64,
1666	.fniv = gen_nand_pg_vec,
1667	.fno = gen_helper_sve_nand_pppp,
1668	.prefer_i64 = true,
1669	};
1670
1671	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1672	return false;
1673	}
1674	return do_pppp_flags(s, a, &op);
1675	}
1676
1677	/*
1678	*** SVE Predicate Misc Group
1679	*/
1680
1681	static bool trans_PTEST(DisasContext s, arg_PTEST a)
1682	{
1683	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
1684	return false;
1685	}
1686	if (sve_access_check(s)) {
1687	int nofs = pred_full_reg_offset(s, a->rn);
1688	int gofs = pred_full_reg_offset(s, a->pg);
1689	int words = DIV_ROUND_UP(pred_full_reg_size(s), 8);
1690
1691	if (words == 1) {
1692	TCGv_i64 pn = tcg_temp_new_i64();
1693	TCGv_i64 pg = tcg_temp_new_i64();
1694
1695	tcg_gen_ld_i64(pn, tcg_env, nofs);
1696	tcg_gen_ld_i64(pg, tcg_env, gofs);
1697	do_predtest1(pn, pg);
1698	} else {
1699	do_predtest(s, nofs, gofs, words);
1700	}
1701	}
1702	return true;
1703	}
1704
1705	/* See the ARM pseudocode DecodePredCount. */
1706	static unsigned decode_pred_count(unsigned fullsz, int pattern, int esz)
1707	{
1708	unsigned elements = fullsz >> esz;
1709	unsigned bound;
1710
1711	switch (pattern) {
1712	case 0x0: /* POW2 */
1713	return pow2floor(elements);
1714	case 0x1: /* VL1 */
1715	case 0x2: /* VL2 */
1716	case 0x3: /* VL3 */
1717	case 0x4: /* VL4 */
1718	case 0x5: /* VL5 */
1719	case 0x6: /* VL6 */
1720	case 0x7: /* VL7 */
1721	case 0x8: /* VL8 */
1722	bound = pattern;
1723	break;
1724	case 0x9: /* VL16 */
1725	case 0xa: /* VL32 */
1726	case 0xb: /* VL64 */
1727	case 0xc: /* VL128 */
1728	case 0xd: /* VL256 */
1729	bound = 16 << (pattern - 9);
1730	break;
1731	case 0x1d: /* MUL4 */
1732	return elements - elements % 4;
1733	case 0x1e: /* MUL3 */
1734	return elements - elements % 3;
1735	case 0x1f: /* ALL */
1736	return elements;
1737	default: /* #uimm5 */
1738	return 0;
1739	}
1740	return elements >= bound ? bound : 0;
1741	}
1742
1743	/* This handles all of the predicate initialization instructions,
1744	* PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1745	* so that decode_pred_count returns 0. For SETFFR, we will have
1746	* set RD == 16 == FFR.
1747	*/
1748	static bool do_predset(DisasContext *s, int esz, int rd, int pat, bool setflag)
1749	{
1750	if (!sve_access_check(s)) {
1751	return true;
1752	}
1753
1754	unsigned fullsz = vec_full_reg_size(s);
1755	unsigned ofs = pred_full_reg_offset(s, rd);
1756	unsigned numelem, setsz, i;
1757	uint64_t word, lastword;
1758	TCGv_i64 t;
1759
1760	numelem = decode_pred_count(fullsz, pat, esz);
1761
1762	/* Determine what we must store into each bit, and how many. */
1763	if (numelem == 0) {
1764	lastword = word = 0;
1765	setsz = fullsz;
1766	} else {
1767	setsz = numelem << esz;
1768	lastword = word = pred_esz_masks[esz];
1769	if (setsz % 64) {
1770	lastword &= MAKE_64BIT_MASK(0, setsz % 64);
1771	}
1772	}
1773
1774	t = tcg_temp_new_i64();
1775	if (fullsz <= 64) {
1776	tcg_gen_movi_i64(t, lastword);
1777	tcg_gen_st_i64(t, tcg_env, ofs);
1778	goto done;
1779	}
1780
1781	if (word == lastword) {
1782	unsigned maxsz = size_for_gvec(fullsz / 8);
1783	unsigned oprsz = size_for_gvec(setsz / 8);
1784
1785	if (oprsz * 8 == setsz) {
1786	tcg_gen_gvec_dup_imm(MO_64, ofs, oprsz, maxsz, word);
1787	goto done;
1788	}
1789	}
1790
1791	setsz /= 8;
1792	fullsz /= 8;
1793
1794	tcg_gen_movi_i64(t, word);
1795	for (i = 0; i < QEMU_ALIGN_DOWN(setsz, 8); i += 8) {
1796	tcg_gen_st_i64(t, tcg_env, ofs + i);
1797	}
1798	if (lastword != word) {
1799	tcg_gen_movi_i64(t, lastword);
1800	tcg_gen_st_i64(t, tcg_env, ofs + i);
1801	i += 8;
1802	}
1803	if (i < fullsz) {
1804	tcg_gen_movi_i64(t, 0);
1805	for (; i < fullsz; i += 8) {
1806	tcg_gen_st_i64(t, tcg_env, ofs + i);
1807	}
1808	}
1809
1810	done:
1811	/* PTRUES */
1812	if (setflag) {
1813	tcg_gen_movi_i32(cpu_NF, -(word != 0));
1814	tcg_gen_movi_i32(cpu_CF, word == 0);
1815	tcg_gen_movi_i32(cpu_VF, 0);
1816	tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1817	}
1818	return true;
1819	}
1820
1821	TRANS_FEAT(PTRUE, aa64_sme_or_sve, do_predset, a->esz, a->rd, a->pat, a->s)
1822
1823	static bool trans_PTRUE_cnt(DisasContext s, arg_PTRUE_cnt a)
1824	{
1825	if (!dc_isar_feature(aa64_sme2_or_sve2p1, s)) {
1826	return false;
1827	}
1828	if (sve_access_check(s)) {
1829	/* Canonical TRUE is 0 count, invert bit, plus element size. */
1830	int val = (1 << 15) \| (1 << a->esz);
1831
1832	/* Write val to the first uint64_t; clear all of the rest. */
1833	tcg_gen_gvec_dup_imm(MO_64, pred_full_reg_offset(s, a->rd),
1834	8, size_for_gvec(pred_full_reg_size(s)), val);
1835	}
1836	return true;
1837	}
1838
1839	/* Note pat == 31 is #all, to set all elements. */
1840	TRANS_FEAT_NONSTREAMING(SETFFR, aa64_sve,
1841	do_predset, 0, FFR_PRED_NUM, 31, false)
1842
1843	/* Note pat == 32 is #unimp, to set no elements. */
1844	TRANS_FEAT(PFALSE, aa64_sme_or_sve, do_predset, 0, a->rd, 32, false)
1845
1846	static bool trans_RDFFR_p(DisasContext s, arg_RDFFR_p a)
1847	{
1848	/* The path through do_pppp_flags is complicated enough to want to avoid
1849	* duplication. Frob the arguments into the form of a predicated AND.
1850	*/
1851	arg_rprr_s alt_a = {
1852	.rd = a->rd, .pg = a->pg, .s = a->s,
1853	.rn = FFR_PRED_NUM, .rm = FFR_PRED_NUM,
1854	};
1855
1856	s->is_nonstreaming = true;
1857	return trans_AND_pppp(s, &alt_a);
1858	}
1859
1860	TRANS_FEAT_NONSTREAMING(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM)
1861	TRANS_FEAT_NONSTREAMING(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn)
1862
1863	static bool do_pfirst_pnext(DisasContext s, arg_rr_esz a,
1864	void (*gen_fn)(TCGv_i32, TCGv_ptr,
1865	TCGv_ptr, TCGv_i32))
1866	{
1867	if (!sve_access_check(s)) {
1868	return true;
1869	}
1870
1871	TCGv_ptr t_pd = tcg_temp_new_ptr();
1872	TCGv_ptr t_pg = tcg_temp_new_ptr();
1873	TCGv_i32 t;
1874	unsigned desc = 0;
1875
1876	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
1877	desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
1878
1879	tcg_gen_addi_ptr(t_pd, tcg_env, pred_full_reg_offset(s, a->rd));
1880	tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->rn));
1881	t = tcg_temp_new_i32();
1882
1883	gen_fn(t, t_pd, t_pg, tcg_constant_i32(desc));
1884
1885	do_pred_flags(t);
1886	return true;
1887	}
1888
1889	TRANS_FEAT(PFIRST, aa64_sme_or_sve, do_pfirst_pnext, a, gen_helper_sve_pfirst)
1890	TRANS_FEAT(PNEXT, aa64_sme_or_sve, do_pfirst_pnext, a, gen_helper_sve_pnext)
1891
1892	/*
1893	*** SVE Element Count Group
1894	*/
1895
1896	/* Perform an inline saturating addition of a 32-bit value within
1897	* a 64-bit register. The second operand is known to be positive,
1898	* which halves the comparisons we must perform to bound the result.
1899	*/
1900	static void do_sat_addsub_32(TCGv_i64 reg, TCGv_i64 val, bool u, bool d)
1901	{
1902	int64_t ibound;
1903
1904	/* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1905	if (u) {
1906	tcg_gen_ext32u_i64(reg, reg);
1907	} else {
1908	tcg_gen_ext32s_i64(reg, reg);
1909	}
1910	if (d) {
1911	tcg_gen_sub_i64(reg, reg, val);
1912	ibound = (u ? 0 : INT32_MIN);
1913	tcg_gen_smax_i64(reg, reg, tcg_constant_i64(ibound));
1914	} else {
1915	tcg_gen_add_i64(reg, reg, val);
1916	ibound = (u ? UINT32_MAX : INT32_MAX);
1917	tcg_gen_smin_i64(reg, reg, tcg_constant_i64(ibound));
1918	}
1919	}
1920
1921	/* Similarly with 64-bit values. */
1922	static void do_sat_addsub_64(TCGv_i64 reg, TCGv_i64 val, bool u, bool d)
1923	{
1924	TCGv_i64 t0 = tcg_temp_new_i64();
1925	TCGv_i64 t2;
1926
1927	if (u) {
1928	if (d) {
1929	tcg_gen_sub_i64(t0, reg, val);
1930	t2 = tcg_constant_i64(0);
1931	tcg_gen_movcond_i64(TCG_COND_LTU, reg, reg, val, t2, t0);
1932	} else {
1933	tcg_gen_add_i64(t0, reg, val);
1934	t2 = tcg_constant_i64(-1);
1935	tcg_gen_movcond_i64(TCG_COND_LTU, reg, t0, reg, t2, t0);
1936	}
1937	} else {
1938	TCGv_i64 t1 = tcg_temp_new_i64();
1939	if (d) {
1940	/* Detect signed overflow for subtraction. */
1941	tcg_gen_xor_i64(t0, reg, val);
1942	tcg_gen_sub_i64(t1, reg, val);
1943	tcg_gen_xor_i64(reg, reg, t1);
1944	tcg_gen_and_i64(t0, t0, reg);
1945
1946	/* Bound the result. */
1947	tcg_gen_movi_i64(reg, INT64_MIN);
1948	t2 = tcg_constant_i64(0);
1949	tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, reg, t1);
1950	} else {
1951	/* Detect signed overflow for addition. */
1952	tcg_gen_xor_i64(t0, reg, val);
1953	tcg_gen_add_i64(reg, reg, val);
1954	tcg_gen_xor_i64(t1, reg, val);
1955	tcg_gen_andc_i64(t0, t1, t0);
1956
1957	/* Bound the result. */
1958	tcg_gen_movi_i64(t1, INT64_MAX);
1959	t2 = tcg_constant_i64(0);
1960	tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, t1, reg);
1961	}
1962	}
1963	}
1964
1965	/* Similarly with a vector and a scalar operand. */
1966	static void do_sat_addsub_vec(DisasContext *s, int esz, int rd, int rn,
1967	TCGv_i64 val, bool u, bool d)
1968	{
1969	unsigned vsz = vec_full_reg_size(s);
1970	TCGv_ptr dptr, nptr;
1971	TCGv_i32 t32, desc;
1972	TCGv_i64 t64;
1973
1974	dptr = tcg_temp_new_ptr();
1975	nptr = tcg_temp_new_ptr();
1976	tcg_gen_addi_ptr(dptr, tcg_env, vec_full_reg_offset(s, rd));
1977	tcg_gen_addi_ptr(nptr, tcg_env, vec_full_reg_offset(s, rn));
1978	desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
1979
1980	switch (esz) {
1981	case MO_8:
1982	t32 = tcg_temp_new_i32();
1983	tcg_gen_extrl_i64_i32(t32, val);
1984	if (d) {
1985	tcg_gen_neg_i32(t32, t32);
1986	}
1987	if (u) {
1988	gen_helper_sve_uqaddi_b(dptr, nptr, t32, desc);
1989	} else {
1990	gen_helper_sve_sqaddi_b(dptr, nptr, t32, desc);
1991	}
1992	break;
1993
1994	case MO_16:
1995	t32 = tcg_temp_new_i32();
1996	tcg_gen_extrl_i64_i32(t32, val);
1997	if (d) {
1998	tcg_gen_neg_i32(t32, t32);
1999	}
2000	if (u) {
2001	gen_helper_sve_uqaddi_h(dptr, nptr, t32, desc);
2002	} else {
2003	gen_helper_sve_sqaddi_h(dptr, nptr, t32, desc);
2004	}
2005	break;
2006
2007	case MO_32:
2008	t64 = tcg_temp_new_i64();
2009	if (d) {
2010	tcg_gen_neg_i64(t64, val);
2011	} else {
2012	tcg_gen_mov_i64(t64, val);
2013	}
2014	if (u) {
2015	gen_helper_sve_uqaddi_s(dptr, nptr, t64, desc);
2016	} else {
2017	gen_helper_sve_sqaddi_s(dptr, nptr, t64, desc);
2018	}
2019	break;
2020
2021	case MO_64:
2022	if (u) {
2023	if (d) {
2024	gen_helper_sve_uqsubi_d(dptr, nptr, val, desc);
2025	} else {
2026	gen_helper_sve_uqaddi_d(dptr, nptr, val, desc);
2027	}
2028	} else if (d) {
2029	t64 = tcg_temp_new_i64();
2030	tcg_gen_neg_i64(t64, val);
2031	gen_helper_sve_sqaddi_d(dptr, nptr, t64, desc);
2032	} else {
2033	gen_helper_sve_sqaddi_d(dptr, nptr, val, desc);
2034	}
2035	break;
2036
2037	default:
2038	g_assert_not_reached();
2039	}
2040	}
2041
2042	static bool trans_CNT_r(DisasContext s, arg_CNT_r a)
2043	{
2044	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2045	return false;
2046	}
2047	if (sve_access_check(s)) {
2048	unsigned fullsz = vec_full_reg_size(s);
2049	unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2050	tcg_gen_movi_i64(cpu_reg(s, a->rd), numelem * a->imm);
2051	}
2052	return true;
2053	}
2054
2055	static bool trans_INCDEC_r(DisasContext s, arg_incdec_cnt a)
2056	{
2057	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2058	return false;
2059	}
2060	if (sve_access_check(s)) {
2061	unsigned fullsz = vec_full_reg_size(s);
2062	unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2063	int inc = numelem * a->imm * (a->d ? -1 : 1);
2064	TCGv_i64 reg = cpu_reg(s, a->rd);
2065
2066	tcg_gen_addi_i64(reg, reg, inc);
2067	}
2068	return true;
2069	}
2070
2071	static bool trans_SINCDEC_r_32(DisasContext s, arg_incdec_cnt a)
2072	{
2073	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2074	return false;
2075	}
2076	if (!sve_access_check(s)) {
2077	return true;
2078	}
2079
2080	unsigned fullsz = vec_full_reg_size(s);
2081	unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2082	int inc = numelem * a->imm;
2083	TCGv_i64 reg = cpu_reg(s, a->rd);
2084
2085	/* Use normal 64-bit arithmetic to detect 32-bit overflow. */
2086	if (inc == 0) {
2087	if (a->u) {
2088	tcg_gen_ext32u_i64(reg, reg);
2089	} else {
2090	tcg_gen_ext32s_i64(reg, reg);
2091	}
2092	} else {
2093	do_sat_addsub_32(reg, tcg_constant_i64(inc), a->u, a->d);
2094	}
2095	return true;
2096	}
2097
2098	static bool trans_SINCDEC_r_64(DisasContext s, arg_incdec_cnt a)
2099	{
2100	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2101	return false;
2102	}
2103	if (!sve_access_check(s)) {
2104	return true;
2105	}
2106
2107	unsigned fullsz = vec_full_reg_size(s);
2108	unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2109	int inc = numelem * a->imm;
2110	TCGv_i64 reg = cpu_reg(s, a->rd);
2111
2112	if (inc != 0) {
2113	do_sat_addsub_64(reg, tcg_constant_i64(inc), a->u, a->d);
2114	}
2115	return true;
2116	}
2117
2118	static bool trans_INCDEC_v(DisasContext s, arg_incdec2_cnt a)
2119	{
2120	if (a->esz == 0 \|\| !dc_isar_feature(aa64_sme_or_sve, s)) {
2121	return false;
2122	}
2123
2124	unsigned fullsz = vec_full_reg_size(s);
2125	unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2126	int inc = numelem * a->imm;
2127
2128	if (inc != 0) {
2129	if (sve_access_check(s)) {
2130	tcg_gen_gvec_adds(a->esz, vec_full_reg_offset(s, a->rd),
2131	vec_full_reg_offset(s, a->rn),
2132	tcg_constant_i64(a->d ? -inc : inc),
2133	fullsz, fullsz);
2134	}
2135	} else {
2136	do_mov_z(s, a->rd, a->rn);
2137	}
2138	return true;
2139	}
2140
2141	static bool trans_SINCDEC_v(DisasContext s, arg_incdec2_cnt a)
2142	{
2143	if (a->esz == 0 \|\| !dc_isar_feature(aa64_sme_or_sve, s)) {
2144	return false;
2145	}
2146
2147	unsigned fullsz = vec_full_reg_size(s);
2148	unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
2149	int inc = numelem * a->imm;
2150
2151	if (inc != 0) {
2152	if (sve_access_check(s)) {
2153	do_sat_addsub_vec(s, a->esz, a->rd, a->rn,
2154	tcg_constant_i64(inc), a->u, a->d);
2155	}
2156	} else {
2157	do_mov_z(s, a->rd, a->rn);
2158	}
2159	return true;
2160	}
2161
2162	/*
2163	*** SVE Bitwise Immediate Group
2164	*/
2165
2166	static bool do_zz_dbm(DisasContext s, arg_rr_dbm a, GVecGen2iFn *gvec_fn)
2167	{
2168	uint64_t imm;
2169	if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
2170	extract32(a->dbm, 0, 6),
2171	extract32(a->dbm, 6, 6))) {
2172	return false;
2173	}
2174	return gen_gvec_fn_zzi(s, gvec_fn, MO_64, a->rd, a->rn, imm);
2175	}
2176
2177	TRANS_FEAT(AND_zzi, aa64_sme_or_sve, do_zz_dbm, a, tcg_gen_gvec_andi)
2178	TRANS_FEAT(ORR_zzi, aa64_sme_or_sve, do_zz_dbm, a, tcg_gen_gvec_ori)
2179	TRANS_FEAT(EOR_zzi, aa64_sme_or_sve, do_zz_dbm, a, tcg_gen_gvec_xori)
2180
2181	static bool trans_DUPM(DisasContext s, arg_DUPM a)
2182	{
2183	uint64_t imm;
2184
2185	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2186	return false;
2187	}
2188	if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
2189	extract32(a->dbm, 0, 6),
2190	extract32(a->dbm, 6, 6))) {
2191	return false;
2192	}
2193	if (sve_access_check(s)) {
2194	do_dupi_z(s, a->rd, imm);
2195	}
2196	return true;
2197	}
2198
2199	/*
2200	*** SVE Integer Wide Immediate - Predicated Group
2201	*/
2202
2203	/* Implement all merging copies. This is used for CPY (immediate),
2204	* FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2205	*/
2206	static void do_cpy_m(DisasContext *s, int esz, int rd, int rn, int pg,
2207	TCGv_i64 val)
2208	{
2209	typedef void gen_cpy(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
2210	static gen_cpy * const fns[4] = {
2211	gen_helper_sve_cpy_m_b, gen_helper_sve_cpy_m_h,
2212	gen_helper_sve_cpy_m_s, gen_helper_sve_cpy_m_d,
2213	};
2214	unsigned vsz = vec_full_reg_size(s);
2215	TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
2216	TCGv_ptr t_zd = tcg_temp_new_ptr();
2217	TCGv_ptr t_zn = tcg_temp_new_ptr();
2218	TCGv_ptr t_pg = tcg_temp_new_ptr();
2219
2220	tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, rd));
2221	tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, rn));
2222	tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
2223
2224	fns[esz](t_zd, t_zn, t_pg, val, desc);
2225	}
2226
2227	static bool trans_FCPY(DisasContext s, arg_FCPY a)
2228	{
2229	if (a->esz == 0 \|\| !dc_isar_feature(aa64_sme_or_sve, s)) {
2230	return false;
2231	}
2232	if (sve_access_check(s)) {
2233	/* Decode the VFP immediate. */
2234	uint64_t imm = vfp_expand_imm(a->esz, a->imm);
2235	do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(imm));
2236	}
2237	return true;
2238	}
2239
2240	static bool trans_CPY_m_i(DisasContext s, arg_rpri_esz a)
2241	{
2242	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2243	return false;
2244	}
2245	if (sve_access_check(s)) {
2246	do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(a->imm));
2247	}
2248	return true;
2249	}
2250
2251	static bool trans_CPY_z_i(DisasContext s, arg_CPY_z_i a)
2252	{
2253	static gen_helper_gvec_2i * const fns[4] = {
2254	gen_helper_sve_cpy_z_b, gen_helper_sve_cpy_z_h,
2255	gen_helper_sve_cpy_z_s, gen_helper_sve_cpy_z_d,
2256	};
2257
2258	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2259	return false;
2260	}
2261	if (sve_access_check(s)) {
2262	unsigned vsz = vec_full_reg_size(s);
2263	tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd),
2264	pred_full_reg_offset(s, a->pg),
2265	tcg_constant_i64(a->imm),
2266	vsz, vsz, 0, fns[a->esz]);
2267	}
2268	return true;
2269	}
2270
2271	/*
2272	*** SVE Permute Extract Group
2273	*/
2274
2275	static bool do_EXT(DisasContext *s, int rd, int rn, int rm, int imm)
2276	{
2277	if (!sve_access_check(s)) {
2278	return true;
2279	}
2280
2281	unsigned vsz = vec_full_reg_size(s);
2282	unsigned n_ofs = imm >= vsz ? 0 : imm;
2283	unsigned n_siz = vsz - n_ofs;
2284	unsigned d = vec_full_reg_offset(s, rd);
2285	unsigned n = vec_full_reg_offset(s, rn);
2286	unsigned m = vec_full_reg_offset(s, rm);
2287
2288	/* Use host vector move insns if we have appropriate sizes
2289	* and no unfortunate overlap.
2290	*/
2291	if (m != d
2292	&& n_ofs == size_for_gvec(n_ofs)
2293	&& n_siz == size_for_gvec(n_siz)
2294	&& (d != n \|\| n_siz <= n_ofs)) {
2295	tcg_gen_gvec_mov(0, d, n + n_ofs, n_siz, n_siz);
2296	if (n_ofs != 0) {
2297	tcg_gen_gvec_mov(0, d + n_siz, m, n_ofs, n_ofs);
2298	}
2299	} else {
2300	tcg_gen_gvec_3_ool(d, n, m, vsz, vsz, n_ofs, gen_helper_sve_ext);
2301	}
2302	return true;
2303	}
2304
2305	TRANS_FEAT(EXT, aa64_sme_or_sve, do_EXT, a->rd, a->rn, a->rm, a->imm)
2306	TRANS_FEAT(EXT_sve2, aa64_sme_or_sve2, do_EXT, a->rd, a->rn, (a->rn + 1) % 32, a->imm)
2307
2308	static bool trans_EXTQ(DisasContext s, arg_EXTQ a)
2309	{
2310	unsigned vl, dofs, sofs0, sofs1, sofs2, imm;
2311
2312	if (!dc_isar_feature(aa64_sme2p1_or_sve2p1, s)) {
2313	return false;
2314	}
2315	if (!sve_access_check(s)) {
2316	return true;
2317	}
2318
2319	imm = a->imm;
2320	if (imm == 0) {
2321	/* So far we never optimize Zdn with MOVPRFX, so zd = zn is a nop. */
2322	return true;
2323	}
2324
2325	vl = vec_full_reg_size(s);
2326	dofs = vec_full_reg_offset(s, a->rd);
2327	sofs2 = vec_full_reg_offset(s, a->rn);
2328
2329	if (imm & 8) {
2330	sofs0 = dofs + 8;
2331	sofs1 = sofs2;
2332	sofs2 += 8;
2333	} else {
2334	sofs0 = dofs;
2335	sofs1 = dofs + 8;
2336	}
2337	imm = (imm & 7) << 3;
2338
2339	for (unsigned i = 0; i < vl; i += 16) {
2340	TCGv_i64 s0 = tcg_temp_new_i64();
2341	TCGv_i64 s1 = tcg_temp_new_i64();
2342	TCGv_i64 s2 = tcg_temp_new_i64();
2343
2344	tcg_gen_ld_i64(s0, tcg_env, sofs0 + i);
2345	tcg_gen_ld_i64(s1, tcg_env, sofs1 + i);
2346	tcg_gen_ld_i64(s2, tcg_env, sofs2 + i);
2347
2348	tcg_gen_extract2_i64(s0, s0, s1, imm);
2349	tcg_gen_extract2_i64(s1, s1, s2, imm);
2350
2351	tcg_gen_st_i64(s0, tcg_env, dofs + i);
2352	tcg_gen_st_i64(s1, tcg_env, dofs + i + 8);
2353	}
2354	return true;
2355	}
2356
2357	/*
2358	*** SVE Permute - Unpredicated Group
2359	*/
2360
2361	static bool trans_DUP_s(DisasContext s, arg_DUP_s a)
2362	{
2363	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2364	return false;
2365	}
2366	if (sve_access_check(s)) {
2367	unsigned vsz = vec_full_reg_size(s);
2368	tcg_gen_gvec_dup_i64(a->esz, vec_full_reg_offset(s, a->rd),
2369	vsz, vsz, cpu_reg_sp(s, a->rn));
2370	}
2371	return true;
2372	}
2373
2374	static bool trans_DUP_x(DisasContext s, arg_DUP_x a)
2375	{
2376	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2377	return false;
2378	}
2379	if ((a->imm & 0x1f) == 0) {
2380	return false;
2381	}
2382	if (sve_access_check(s)) {
2383	unsigned vsz = vec_full_reg_size(s);
2384	unsigned dofs = vec_full_reg_offset(s, a->rd);
2385	unsigned esz, index;
2386
2387	esz = ctz32(a->imm);
2388	index = a->imm >> (esz + 1);
2389
2390	if ((index << esz) < vsz) {
2391	unsigned nofs = vec_reg_offset(s, a->rn, index, esz);
2392	tcg_gen_gvec_dup_mem(esz, dofs, nofs, vsz, vsz);
2393	} else {
2394	/*
2395	* While dup_mem handles 128-bit elements, dup_imm does not.
2396	* Thankfully element size doesn't matter for splatting zero.
2397	*/
2398	tcg_gen_gvec_dup_imm(MO_64, dofs, vsz, vsz, 0);
2399	}
2400	}
2401	return true;
2402	}
2403
2404	static bool trans_DUPQ(DisasContext s, arg_DUPQ a)
2405	{
2406	unsigned vl, dofs, nofs;
2407
2408	if (!dc_isar_feature(aa64_sme2p1_or_sve2p1, s)) {
2409	return false;
2410	}
2411	if (!sve_access_check(s)) {
2412	return true;
2413	}
2414
2415	vl = vec_full_reg_size(s);
2416	dofs = vec_full_reg_offset(s, a->rd);
2417	nofs = vec_reg_offset(s, a->rn, a->imm, a->esz);
2418
2419	for (unsigned i = 0; i < vl; i += 16) {
2420	tcg_gen_gvec_dup_mem(a->esz, dofs + i, nofs + i, 16, 16);
2421	}
2422	return true;
2423	}
2424
2425	static void do_insr_i64(DisasContext s, arg_rrr_esz a, TCGv_i64 val)
2426	{
2427	typedef void gen_insr(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
2428	static gen_insr * const fns[4] = {
2429	gen_helper_sve_insr_b, gen_helper_sve_insr_h,
2430	gen_helper_sve_insr_s, gen_helper_sve_insr_d,
2431	};
2432	unsigned vsz = vec_full_reg_size(s);
2433	TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
2434	TCGv_ptr t_zd = tcg_temp_new_ptr();
2435	TCGv_ptr t_zn = tcg_temp_new_ptr();
2436
2437	tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, a->rd));
2438	tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn));
2439
2440	fns[a->esz](t_zd, t_zn, val, desc);
2441	}
2442
2443	static bool trans_INSR_f(DisasContext s, arg_rrr_esz a)
2444	{
2445	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2446	return false;
2447	}
2448	if (sve_access_check(s)) {
2449	TCGv_i64 t = tcg_temp_new_i64();
2450	tcg_gen_ld_i64(t, tcg_env, vec_reg_offset(s, a->rm, 0, MO_64));
2451	do_insr_i64(s, a, t);
2452	}
2453	return true;
2454	}
2455
2456	static bool trans_INSR_r(DisasContext s, arg_rrr_esz a)
2457	{
2458	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
2459	return false;
2460	}
2461	if (sve_access_check(s)) {
2462	do_insr_i64(s, a, cpu_reg(s, a->rm));
2463	}
2464	return true;
2465	}
2466
2467	static gen_helper_gvec_2 * const rev_fns[4] = {
2468	gen_helper_sve_rev_b, gen_helper_sve_rev_h,
2469	gen_helper_sve_rev_s, gen_helper_sve_rev_d
2470	};
2471	TRANS_FEAT(REV_v, aa64_sme_or_sve, gen_gvec_ool_zz, rev_fns[a->esz], a->rd, a->rn, 0)
2472
2473	static gen_helper_gvec_3 * const sve_tbl_fns[4] = {
2474	gen_helper_sve_tbl_b, gen_helper_sve_tbl_h,
2475	gen_helper_sve_tbl_s, gen_helper_sve_tbl_d
2476	};
2477	TRANS_FEAT(TBL, aa64_sme_or_sve, gen_gvec_ool_arg_zzz, sve_tbl_fns[a->esz], a, 0)
2478
2479	static gen_helper_gvec_4 * const sve2_tbl_fns[4] = {
2480	gen_helper_sve2_tbl_b, gen_helper_sve2_tbl_h,
2481	gen_helper_sve2_tbl_s, gen_helper_sve2_tbl_d
2482	};
2483	TRANS_FEAT(TBL_sve2, aa64_sme_or_sve2, gen_gvec_ool_zzzz, sve2_tbl_fns[a->esz],
2484	a->rd, a->rn, (a->rn + 1) % 32, a->rm, 0)
2485
2486	static gen_helper_gvec_3 * const tblq_fns[4] = {
2487	gen_helper_sve2p1_tblq_b, gen_helper_sve2p1_tblq_h,
2488	gen_helper_sve2p1_tblq_s, gen_helper_sve2p1_tblq_d
2489	};
2490	TRANS_FEAT(TBLQ, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zzz,
2491	tblq_fns[a->esz], a, 0)
2492
2493	static gen_helper_gvec_3 * const tbx_fns[4] = {
2494	gen_helper_sve2_tbx_b, gen_helper_sve2_tbx_h,
2495	gen_helper_sve2_tbx_s, gen_helper_sve2_tbx_d
2496	};
2497	TRANS_FEAT(TBX, aa64_sme_or_sve2, gen_gvec_ool_arg_zzz, tbx_fns[a->esz], a, 0)
2498
2499	static gen_helper_gvec_3 * const tbxq_fns[4] = {
2500	gen_helper_sve2p1_tbxq_b, gen_helper_sve2p1_tbxq_h,
2501	gen_helper_sve2p1_tbxq_s, gen_helper_sve2p1_tbxq_d
2502	};
2503	TRANS_FEAT(TBXQ, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zzz,
2504	tbxq_fns[a->esz], a, 0)
2505
2506	static bool trans_PMOV_pv(DisasContext s, arg_PMOV_pv a)
2507	{
2508	static gen_helper_gvec_2 * const fns[4] = {
2509	NULL, gen_helper_pmov_pv_h,
2510	gen_helper_pmov_pv_s, gen_helper_pmov_pv_d
2511	};
2512	unsigned vl, pl, vofs, pofs;
2513	TCGv_i64 tmp;
2514
2515	if (!dc_isar_feature(aa64_sme2p1_or_sve2p1, s)) {
2516	return false;
2517	}
2518	if (!sve_access_check(s)) {
2519	return true;
2520	}
2521
2522	vl = vec_full_reg_size(s);
2523	if (a->esz != MO_8) {
2524	tcg_gen_gvec_2_ool(pred_full_reg_offset(s, a->rd),
2525	vec_full_reg_offset(s, a->rn),
2526	vl, vl, a->imm, fns[a->esz]);
2527	return true;
2528	}
2529
2530	/*
2531	* Copy the low PL bytes from vector Zn, zero-extending to a
2532	* multiple of 8 bytes, so that Pd is properly cleared.
2533	*/
2534
2535	pl = vl / 8;
2536	pofs = pred_full_reg_offset(s, a->rd);
2537	vofs = vec_full_reg_offset(s, a->rn);
2538
2539	QEMU_BUILD_BUG_ON(sizeof(ARMPredicateReg) != 32);
2540	for (unsigned i = 32; i >= 8; i >>= 1) {
2541	if (pl & i) {
2542	tcg_gen_gvec_mov(MO_64, pofs, vofs, i, i);
2543	pofs += i;
2544	vofs += i;
2545	}
2546	}
2547	switch (pl & 7) {
2548	case 0:
2549	return true;
2550	case 2:
2551	tmp = tcg_temp_new_i64();
2552	tcg_gen_ld16u_i64(tmp, tcg_env, vofs + (HOST_BIG_ENDIAN ? 6 : 0));
2553	break;
2554	case 4:
2555	tmp = tcg_temp_new_i64();
2556	tcg_gen_ld32u_i64(tmp, tcg_env, vofs + (HOST_BIG_ENDIAN ? 4 : 0));
2557	break;
2558	case 6:
2559	tmp = tcg_temp_new_i64();
2560	tcg_gen_ld_i64(tmp, tcg_env, vofs);
2561	tcg_gen_extract_i64(tmp, tmp, 0, 48);
2562	break;
2563	default:
2564	g_assert_not_reached();
2565	}
2566	tcg_gen_st_i64(tmp, tcg_env, pofs);
2567	return true;
2568	}
2569
2570	static bool trans_PMOV_vp(DisasContext s, arg_PMOV_pv a)
2571	{
2572	static gen_helper_gvec_2 * const fns[4] = {
2573	NULL, gen_helper_pmov_vp_h,
2574	gen_helper_pmov_vp_s, gen_helper_pmov_vp_d
2575	};
2576	unsigned vl;
2577
2578	if (!dc_isar_feature(aa64_sme2p1_or_sve2p1, s)) {
2579	return false;
2580	}
2581	if (!sve_access_check(s)) {
2582	return true;
2583	}
2584
2585	vl = vec_full_reg_size(s);
2586
2587	if (a->esz == MO_8) {
2588	/*
2589	* The low PL bytes are copied from Pn to Zd unchanged.
2590	* We know that the unused portion of Pn is zero, and
2591	* that imm == 0, so the balance of Zd must be zeroed.
2592	*/
2593	tcg_gen_gvec_mov(MO_64, vec_full_reg_offset(s, a->rd),
2594	pred_full_reg_offset(s, a->rn),
2595	size_for_gvec(vl / 8), vl);
2596	} else {
2597	tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd),
2598	pred_full_reg_offset(s, a->rn),
2599	vl, vl, a->imm, fns[a->esz]);
2600	}
2601	return true;
2602	}
2603
2604	static bool trans_UNPK(DisasContext s, arg_UNPK a)
2605	{
2606	static gen_helper_gvec_2 * const fns[4][2] = {
2607	{ NULL, NULL },
2608	{ gen_helper_sve_sunpk_h, gen_helper_sve_uunpk_h },
2609	{ gen_helper_sve_sunpk_s, gen_helper_sve_uunpk_s },
2610	{ gen_helper_sve_sunpk_d, gen_helper_sve_uunpk_d },
2611	};
2612
2613	if (a->esz == 0 \|\| !dc_isar_feature(aa64_sme_or_sve, s)) {
2614	return false;
2615	}
2616	if (sve_access_check(s)) {
2617	unsigned vsz = vec_full_reg_size(s);
2618	tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd),
2619	vec_full_reg_offset(s, a->rn)
2620	+ (a->h ? vsz / 2 : 0),
2621	vsz, vsz, 0, fns[a->esz][a->u]);
2622	}
2623	return true;
2624	}
2625
2626	/*
2627	*** SVE Permute - Predicates Group
2628	*/
2629
2630	static bool do_perm_pred3(DisasContext s, arg_rrr_esz a, bool high_odd,
2631	gen_helper_gvec_3 *fn)
2632	{
2633	if (!sve_access_check(s)) {
2634	return true;
2635	}
2636
2637	unsigned vsz = pred_full_reg_size(s);
2638
2639	TCGv_ptr t_d = tcg_temp_new_ptr();
2640	TCGv_ptr t_n = tcg_temp_new_ptr();
2641	TCGv_ptr t_m = tcg_temp_new_ptr();
2642	uint32_t desc = 0;
2643
2644	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz);
2645	desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
2646	desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd);
2647
2648	tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd));
2649	tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn));
2650	tcg_gen_addi_ptr(t_m, tcg_env, pred_full_reg_offset(s, a->rm));
2651
2652	fn(t_d, t_n, t_m, tcg_constant_i32(desc));
2653	return true;
2654	}
2655
2656	static bool do_perm_pred2(DisasContext s, arg_rr_esz a, bool high_odd,
2657	gen_helper_gvec_2 *fn)
2658	{
2659	if (!sve_access_check(s)) {
2660	return true;
2661	}
2662
2663	unsigned vsz = pred_full_reg_size(s);
2664	TCGv_ptr t_d = tcg_temp_new_ptr();
2665	TCGv_ptr t_n = tcg_temp_new_ptr();
2666	uint32_t desc = 0;
2667
2668	tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd));
2669	tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn));
2670
2671	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz);
2672	desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
2673	desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd);
2674
2675	fn(t_d, t_n, tcg_constant_i32(desc));
2676	return true;
2677	}
2678
2679	TRANS_FEAT(ZIP1_p, aa64_sme_or_sve, do_perm_pred3, a, 0, gen_helper_sve_zip_p)
2680	TRANS_FEAT(ZIP2_p, aa64_sme_or_sve, do_perm_pred3, a, 1, gen_helper_sve_zip_p)
2681	TRANS_FEAT(UZP1_p, aa64_sme_or_sve, do_perm_pred3, a, 0, gen_helper_sve_uzp_p)
2682	TRANS_FEAT(UZP2_p, aa64_sme_or_sve, do_perm_pred3, a, 1, gen_helper_sve_uzp_p)
2683	TRANS_FEAT(TRN1_p, aa64_sme_or_sve, do_perm_pred3, a, 0, gen_helper_sve_trn_p)
2684	TRANS_FEAT(TRN2_p, aa64_sme_or_sve, do_perm_pred3, a, 1, gen_helper_sve_trn_p)
2685
2686	TRANS_FEAT(REV_p, aa64_sme_or_sve, do_perm_pred2, a, 0, gen_helper_sve_rev_p)
2687	TRANS_FEAT(PUNPKLO, aa64_sme_or_sve, do_perm_pred2, a, 0, gen_helper_sve_punpk_p)
2688	TRANS_FEAT(PUNPKHI, aa64_sme_or_sve, do_perm_pred2, a, 1, gen_helper_sve_punpk_p)
2689
2690	/*
2691	*** SVE Permute - Interleaving Group
2692	*/
2693
2694	static bool do_interleave_q(DisasContext s, gen_helper_gvec_3 fn,
2695	arg_rrr_esz *a, int data)
2696	{
2697	if (sve_access_check(s)) {
2698	unsigned vsz = vec_full_reg_size(s);
2699	if (vsz < 32) {
2700	unallocated_encoding(s);
2701	} else {
2702	tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd),
2703	vec_full_reg_offset(s, a->rn),
2704	vec_full_reg_offset(s, a->rm),
2705	vsz, vsz, data, fn);
2706	}
2707	}
2708	return true;
2709	}
2710
2711	static gen_helper_gvec_3 * const zip_fns[4] = {
2712	gen_helper_sve_zip_b, gen_helper_sve_zip_h,
2713	gen_helper_sve_zip_s, gen_helper_sve_zip_d,
2714	};
2715	TRANS_FEAT(ZIP1_z, aa64_sme_or_sve, gen_gvec_ool_arg_zzz,
2716	zip_fns[a->esz], a, 0)
2717	TRANS_FEAT(ZIP2_z, aa64_sme_or_sve, gen_gvec_ool_arg_zzz,
2718	zip_fns[a->esz], a, vec_full_reg_size(s) / 2)
2719
2720	TRANS_FEAT_NONSTREAMING(ZIP1_q, aa64_sve_f64mm, do_interleave_q,
2721	gen_helper_sve2_zip_q, a, 0)
2722	TRANS_FEAT_NONSTREAMING(ZIP2_q, aa64_sve_f64mm, do_interleave_q,
2723	gen_helper_sve2_zip_q, a,
2724	QEMU_ALIGN_DOWN(vec_full_reg_size(s), 32) / 2)
2725
2726	static gen_helper_gvec_3 * const zipq_fns[4] = {
2727	gen_helper_sve2p1_zipq_b, gen_helper_sve2p1_zipq_h,
2728	gen_helper_sve2p1_zipq_s, gen_helper_sve2p1_zipq_d,
2729	};
2730	TRANS_FEAT(ZIPQ1, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zzz,
2731	zipq_fns[a->esz], a, 0)
2732	TRANS_FEAT(ZIPQ2, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zzz,
2733	zipq_fns[a->esz], a, 16 / 2)
2734
2735	static gen_helper_gvec_3 * const uzp_fns[4] = {
2736	gen_helper_sve_uzp_b, gen_helper_sve_uzp_h,
2737	gen_helper_sve_uzp_s, gen_helper_sve_uzp_d,
2738	};
2739	TRANS_FEAT(UZP1_z, aa64_sme_or_sve, gen_gvec_ool_arg_zzz,
2740	uzp_fns[a->esz], a, 0)
2741	TRANS_FEAT(UZP2_z, aa64_sme_or_sve, gen_gvec_ool_arg_zzz,
2742	uzp_fns[a->esz], a, 1 << a->esz)
2743
2744	TRANS_FEAT_NONSTREAMING(UZP1_q, aa64_sve_f64mm, do_interleave_q,
2745	gen_helper_sve2_uzp_q, a, 0)
2746	TRANS_FEAT_NONSTREAMING(UZP2_q, aa64_sve_f64mm, do_interleave_q,
2747	gen_helper_sve2_uzp_q, a, 16)
2748
2749	static gen_helper_gvec_3 * const uzpq_fns[4] = {
2750	gen_helper_sve2p1_uzpq_b, gen_helper_sve2p1_uzpq_h,
2751	gen_helper_sve2p1_uzpq_s, gen_helper_sve2p1_uzpq_d,
2752	};
2753	TRANS_FEAT(UZPQ1, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zzz,
2754	uzpq_fns[a->esz], a, 0)
2755	TRANS_FEAT(UZPQ2, aa64_sme2p1_or_sve2p1, gen_gvec_ool_arg_zzz,
2756	uzpq_fns[a->esz], a, 1 << a->esz)
2757
2758	static gen_helper_gvec_3 * const trn_fns[4] = {
2759	gen_helper_sve_trn_b, gen_helper_sve_trn_h,
2760	gen_helper_sve_trn_s, gen_helper_sve_trn_d,
2761	};
2762
2763	TRANS_FEAT(TRN1_z, aa64_sme_or_sve, gen_gvec_ool_arg_zzz,
2764	trn_fns[a->esz], a, 0)
2765	TRANS_FEAT(TRN2_z, aa64_sme_or_sve, gen_gvec_ool_arg_zzz,
2766	trn_fns[a->esz], a, 1 << a->esz)
2767
2768	TRANS_FEAT_NONSTREAMING(TRN1_q, aa64_sve_f64mm, do_interleave_q,
2769	gen_helper_sve2_trn_q, a, 0)
2770	TRANS_FEAT_NONSTREAMING(TRN2_q, aa64_sve_f64mm, do_interleave_q,
2771	gen_helper_sve2_trn_q, a, 16)
2772
2773	/*
2774	*** SVE Permute Vector - Predicated Group
2775	*/
2776
2777	static bool trans_COMPACT(DisasContext s, arg_COMPACT a)
2778	{
2779	static gen_helper_gvec_3 * const fns[4] = {
2780	gen_helper_sve_compact_b, gen_helper_sve_compact_h,
2781	gen_helper_sve_compact_s, gen_helper_sve_compact_d
2782	};
2783
2784	if (!dc_isar_feature(aa64_sme2p2, s)) {
2785	if (!(a->esz >= MO_32
2786	? dc_isar_feature(aa64_sve, s)
2787	: dc_isar_feature(aa64_sve2p2, s))) {
2788	return false;
2789	}
2790	s->is_nonstreaming = true;
2791	}
2792	return gen_gvec_ool_arg_zpz(s, fns[a->esz], a, 0);
2793	}
2794
2795	/* Call the helper that computes the ARM LastActiveElement pseudocode
2796	* function, scaled by the element size. This includes the not found
2797	* indication; e.g. not found for esz=3 is -8.
2798	*/
2799	static void find_last_active(DisasContext *s, TCGv_i32 ret, int esz, int pg)
2800	{
2801	/* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2802	* round up, as we do elsewhere, because we need the exact size.
2803	*/
2804	TCGv_ptr t_p = tcg_temp_new_ptr();
2805	unsigned desc = 0;
2806
2807	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
2808	desc = FIELD_DP32(desc, PREDDESC, ESZ, esz);
2809
2810	tcg_gen_addi_ptr(t_p, tcg_env, pred_full_reg_offset(s, pg));
2811
2812	gen_helper_sve_last_active_element(ret, t_p, tcg_constant_i32(desc));
2813	}
2814
2815	/* Increment LAST to the offset of the next element in the vector,
2816	* wrapping around to 0.
2817	*/
2818	static void incr_last_active(DisasContext *s, TCGv_i32 last, int esz)
2819	{
2820	unsigned vsz = vec_full_reg_size(s);
2821
2822	tcg_gen_addi_i32(last, last, 1 << esz);
2823	if (is_power_of_2(vsz)) {
2824	tcg_gen_andi_i32(last, last, vsz - 1);
2825	} else {
2826	TCGv_i32 max = tcg_constant_i32(vsz);
2827	TCGv_i32 zero = tcg_constant_i32(0);
2828	tcg_gen_movcond_i32(TCG_COND_GEU, last, last, max, zero, last);
2829	}
2830	}
2831
2832	/* If LAST < 0, set LAST to the offset of the last element in the vector. */
2833	static void wrap_last_active(DisasContext *s, TCGv_i32 last, int esz)
2834	{
2835	unsigned vsz = vec_full_reg_size(s);
2836
2837	if (is_power_of_2(vsz)) {
2838	tcg_gen_andi_i32(last, last, vsz - 1);
2839	} else {
2840	TCGv_i32 max = tcg_constant_i32(vsz - (1 << esz));
2841	TCGv_i32 zero = tcg_constant_i32(0);
2842	tcg_gen_movcond_i32(TCG_COND_LT, last, last, zero, max, last);
2843	}
2844	}
2845
2846	/* Load an unsigned element of ESZ from BASE+OFS. */
2847	static TCGv_i64 load_esz(TCGv_ptr base, int ofs, int esz)
2848	{
2849	TCGv_i64 r = tcg_temp_new_i64();
2850
2851	switch (esz) {
2852	case 0:
2853	tcg_gen_ld8u_i64(r, base, ofs);
2854	break;
2855	case 1:
2856	tcg_gen_ld16u_i64(r, base, ofs);
2857	break;
2858	case 2:
2859	tcg_gen_ld32u_i64(r, base, ofs);
2860	break;
2861	case 3:
2862	tcg_gen_ld_i64(r, base, ofs);
2863	break;
2864	default:
2865	g_assert_not_reached();
2866	}
2867	return r;
2868	}
2869
2870	/* Load an unsigned element of ESZ from RM[LAST]. */
2871	static TCGv_i64 load_last_active(DisasContext *s, TCGv_i32 last,
2872	int rm, int esz)
2873	{
2874	TCGv_ptr p = tcg_temp_new_ptr();
2875
2876	/* Convert offset into vector into offset into ENV.
2877	* The final adjustment for the vector register base
2878	* is added via constant offset to the load.
2879	*/
2880	#if HOST_BIG_ENDIAN
2881	/* Adjust for element ordering. See vec_reg_offset. */
2882	if (esz < 3) {
2883	tcg_gen_xori_i32(last, last, 8 - (1 << esz));
2884	}
2885	#endif
2886	tcg_gen_ext_i32_ptr(p, last);
2887	tcg_gen_add_ptr(p, p, tcg_env);
2888
2889	return load_esz(p, vec_full_reg_offset(s, rm), esz);
2890	}
2891
2892	/* Compute CLAST for a Zreg. */
2893	static bool do_clast_vector(DisasContext s, arg_rprr_esz a, bool before)
2894	{
2895	TCGv_i32 last;
2896	TCGLabel *over;
2897	TCGv_i64 ele;
2898	unsigned vsz, esz = a->esz;
2899
2900	if (!sve_access_check(s)) {
2901	return true;
2902	}
2903
2904	last = tcg_temp_new_i32();
2905	over = gen_new_label();
2906
2907	find_last_active(s, last, esz, a->pg);
2908
2909	/* There is of course no movcond for a 2048-bit vector,
2910	* so we must branch over the actual store.
2911	*/
2912	tcg_gen_brcondi_i32(TCG_COND_LT, last, 0, over);
2913
2914	if (!before) {
2915	incr_last_active(s, last, esz);
2916	}
2917
2918	ele = load_last_active(s, last, a->rm, esz);
2919
2920	vsz = vec_full_reg_size(s);
2921	tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), vsz, vsz, ele);
2922
2923	/* If this insn used MOVPRFX, we may need a second move. */
2924	if (a->rd != a->rn) {
2925	TCGLabel *done = gen_new_label();
2926	tcg_gen_br(done);
2927
2928	gen_set_label(over);
2929	do_mov_z(s, a->rd, a->rn);
2930
2931	gen_set_label(done);
2932	} else {
2933	gen_set_label(over);
2934	}
2935	return true;
2936	}
2937
2938	TRANS_FEAT(CLASTA_z, aa64_sme_or_sve, do_clast_vector, a, false)
2939	TRANS_FEAT(CLASTB_z, aa64_sme_or_sve, do_clast_vector, a, true)
2940
2941	/* Compute CLAST for a scalar. */
2942	static void do_clast_scalar(DisasContext *s, int esz, int pg, int rm,
2943	bool before, TCGv_i64 reg_val)
2944	{
2945	TCGv_i32 last = tcg_temp_new_i32();
2946	TCGv_i64 ele, cmp;
2947
2948	find_last_active(s, last, esz, pg);
2949
2950	/* Extend the original value of last prior to incrementing. */
2951	cmp = tcg_temp_new_i64();
2952	tcg_gen_ext_i32_i64(cmp, last);
2953
2954	if (!before) {
2955	incr_last_active(s, last, esz);
2956	}
2957
2958	/* The conceit here is that while last < 0 indicates not found, after
2959	* adjusting for tcg_env->vfp.zregs[rm], it is still a valid address
2960	* from which we can load garbage. We then discard the garbage with
2961	* a conditional move.
2962	*/
2963	ele = load_last_active(s, last, rm, esz);
2964
2965	tcg_gen_movcond_i64(TCG_COND_GE, reg_val, cmp, tcg_constant_i64(0),
2966	ele, reg_val);
2967	}
2968
2969	/* Compute CLAST for a Vreg. */
2970	static bool do_clast_fp(DisasContext s, arg_rpr_esz a, bool before)
2971	{
2972	if (sve_access_check(s)) {
2973	int esz = a->esz;
2974	int ofs = vec_reg_offset(s, a->rd, 0, esz);
2975	TCGv_i64 reg = load_esz(tcg_env, ofs, esz);
2976
2977	do_clast_scalar(s, esz, a->pg, a->rn, before, reg);
2978	write_fp_dreg(s, a->rd, reg);
2979	}
2980	return true;
2981	}
2982
2983	TRANS_FEAT(CLASTA_v, aa64_sme_or_sve, do_clast_fp, a, false)
2984	TRANS_FEAT(CLASTB_v, aa64_sme_or_sve, do_clast_fp, a, true)
2985
2986	/* Compute CLAST for a Xreg. */
2987	static bool do_clast_general(DisasContext s, arg_rpr_esz a, bool before)
2988	{
2989	TCGv_i64 reg;
2990
2991	if (!sve_access_check(s)) {
2992	return true;
2993	}
2994
2995	reg = cpu_reg(s, a->rd);
2996	switch (a->esz) {
2997	case 0:
2998	tcg_gen_ext8u_i64(reg, reg);
2999	break;
3000	case 1:
3001	tcg_gen_ext16u_i64(reg, reg);
3002	break;
3003	case 2:
3004	tcg_gen_ext32u_i64(reg, reg);
3005	break;
3006	case 3:
3007	break;
3008	default:
3009	g_assert_not_reached();
3010	}
3011
3012	do_clast_scalar(s, a->esz, a->pg, a->rn, before, reg);
3013	return true;
3014	}
3015
3016	TRANS_FEAT(CLASTA_r, aa64_sme_or_sve, do_clast_general, a, false)
3017	TRANS_FEAT(CLASTB_r, aa64_sme_or_sve, do_clast_general, a, true)
3018
3019	/* Compute LAST for a scalar. */
3020	static TCGv_i64 do_last_scalar(DisasContext *s, int esz,
3021	int pg, int rm, bool before)
3022	{
3023	TCGv_i32 last = tcg_temp_new_i32();
3024
3025	find_last_active(s, last, esz, pg);
3026	if (before) {
3027	wrap_last_active(s, last, esz);
3028	} else {
3029	incr_last_active(s, last, esz);
3030	}
3031
3032	return load_last_active(s, last, rm, esz);
3033	}
3034
3035	/* Compute LAST for a Vreg. */
3036	static bool do_last_fp(DisasContext s, arg_rpr_esz a, bool before)
3037	{
3038	if (sve_access_check(s)) {
3039	TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before);
3040	write_fp_dreg(s, a->rd, val);
3041	}
3042	return true;
3043	}
3044
3045	TRANS_FEAT(LASTA_v, aa64_sme_or_sve, do_last_fp, a, false)
3046	TRANS_FEAT(LASTB_v, aa64_sme_or_sve, do_last_fp, a, true)
3047
3048	/* Compute LAST for a Xreg. */
3049	static bool do_last_general(DisasContext s, arg_rpr_esz a, bool before)
3050	{
3051	if (sve_access_check(s)) {
3052	TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before);
3053	tcg_gen_mov_i64(cpu_reg(s, a->rd), val);
3054	}
3055	return true;
3056	}
3057
3058	TRANS_FEAT(LASTA_r, aa64_sme_or_sve, do_last_general, a, false)
3059	TRANS_FEAT(LASTB_r, aa64_sme_or_sve, do_last_general, a, true)
3060
3061	static bool trans_CPY_m_r(DisasContext s, arg_rpr_esz a)
3062	{
3063	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3064	return false;
3065	}
3066	if (sve_access_check(s)) {
3067	do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, cpu_reg_sp(s, a->rn));
3068	}
3069	return true;
3070	}
3071
3072	static bool trans_CPY_m_v(DisasContext s, arg_rpr_esz a)
3073	{
3074	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3075	return false;
3076	}
3077	if (sve_access_check(s)) {
3078	int ofs = vec_reg_offset(s, a->rn, 0, a->esz);
3079	TCGv_i64 t = load_esz(tcg_env, ofs, a->esz);
3080	do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, t);
3081	}
3082	return true;
3083	}
3084
3085	static gen_helper_gvec_3 * const revb_fns[4] = {
3086	NULL, gen_helper_sve_revb_h,
3087	gen_helper_sve_revb_s, gen_helper_sve_revb_d,
3088	};
3089	TRANS_FEAT(REVB_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
3090	revb_fns[a->esz], a, 0)
3091	TRANS_FEAT(REVB_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
3092	revb_fns[a->esz], a, 1)
3093
3094	static gen_helper_gvec_3 * const revh_fns[4] = {
3095	NULL, NULL, gen_helper_sve_revh_s, gen_helper_sve_revh_d,
3096	};
3097	TRANS_FEAT(REVH_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
3098	revh_fns[a->esz], a, 0)
3099	TRANS_FEAT(REVH_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
3100	revh_fns[a->esz], a, 1)
3101
3102	TRANS_FEAT(REVW_m, aa64_sme_or_sve, gen_gvec_ool_arg_zpz,
3103	a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 0)
3104	TRANS_FEAT(REVW_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
3105	a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 1)
3106
3107	TRANS_FEAT(REVD_m, aa64_sme_or_sve2p1, gen_gvec_ool_arg_zpz,
3108	gen_helper_sme_revd_q, a, 0)
3109	TRANS_FEAT(REVD_z, aa64_sme2p2_or_sve2p2, gen_gvec_ool_arg_zpz,
3110	gen_helper_sme_revd_q, a, 1)
3111
3112	TRANS_FEAT(SPLICE, aa64_sme_or_sve, gen_gvec_ool_arg_zpzz,
3113	gen_helper_sve_splice, a, a->esz)
3114
3115	TRANS_FEAT(SPLICE_sve2, aa64_sme_or_sve2, gen_gvec_ool_zzzp, gen_helper_sve_splice,
3116	a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz)
3117
3118	/*
3119	*** SVE Integer Compare - Vectors Group
3120	*/
3121
3122	static bool do_ppzz_flags(DisasContext s, arg_rprr_esz a,
3123	gen_helper_gvec_flags_4 *gen_fn)
3124	{
3125	TCGv_ptr pd, zn, zm, pg;
3126	unsigned vsz;
3127	TCGv_i32 t;
3128
3129	if (gen_fn == NULL) {
3130	return false;
3131	}
3132	if (!sve_access_check(s)) {
3133	return true;
3134	}
3135
3136	vsz = vec_full_reg_size(s);
3137	t = tcg_temp_new_i32();
3138	pd = tcg_temp_new_ptr();
3139	zn = tcg_temp_new_ptr();
3140	zm = tcg_temp_new_ptr();
3141	pg = tcg_temp_new_ptr();
3142
3143	tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd));
3144	tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn));
3145	tcg_gen_addi_ptr(zm, tcg_env, vec_full_reg_offset(s, a->rm));
3146	tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg));
3147
3148	gen_fn(t, pd, zn, zm, pg, tcg_constant_i32(simd_desc(vsz, vsz, 0)));
3149
3150	do_pred_flags(t);
3151	return true;
3152	}
3153
3154	#define DO_PPZZ(NAME, name) \
3155	static gen_helper_gvec_flags_4 * const name##_ppzz_fns[4] = { \
3156	gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
3157	gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
3158	}; \
3159	TRANS_FEAT(NAME##_ppzz, aa64_sme_or_sve, do_ppzz_flags, \
3160	a, name##_ppzz_fns[a->esz])
3161
3162	DO_PPZZ(CMPEQ, cmpeq)
3163	DO_PPZZ(CMPNE, cmpne)
3164	DO_PPZZ(CMPGT, cmpgt)
3165	DO_PPZZ(CMPGE, cmpge)
3166	DO_PPZZ(CMPHI, cmphi)
3167	DO_PPZZ(CMPHS, cmphs)
3168
3169	#undef DO_PPZZ
3170
3171	#define DO_PPZW(NAME, name) \
3172	static gen_helper_gvec_flags_4 * const name##_ppzw_fns[4] = { \
3173	gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
3174	gen_helper_sve_##name##_ppzw_s, NULL \
3175	}; \
3176	TRANS_FEAT(NAME##_ppzw, aa64_sme_or_sve, do_ppzz_flags, \
3177	a, name##_ppzw_fns[a->esz])
3178
3179	DO_PPZW(CMPEQ, cmpeq)
3180	DO_PPZW(CMPNE, cmpne)
3181	DO_PPZW(CMPGT, cmpgt)
3182	DO_PPZW(CMPGE, cmpge)
3183	DO_PPZW(CMPHI, cmphi)
3184	DO_PPZW(CMPHS, cmphs)
3185	DO_PPZW(CMPLT, cmplt)
3186	DO_PPZW(CMPLE, cmple)
3187	DO_PPZW(CMPLO, cmplo)
3188	DO_PPZW(CMPLS, cmpls)
3189
3190	#undef DO_PPZW
3191
3192	/*
3193	*** SVE Integer Compare - Immediate Groups
3194	*/
3195
3196	static bool do_ppzi_flags(DisasContext s, arg_rpri_esz a,
3197	gen_helper_gvec_flags_3 *gen_fn)
3198	{
3199	TCGv_ptr pd, zn, pg;
3200	unsigned vsz;
3201	TCGv_i32 t;
3202
3203	if (gen_fn == NULL) {
3204	return false;
3205	}
3206	if (!sve_access_check(s)) {
3207	return true;
3208	}
3209
3210	vsz = vec_full_reg_size(s);
3211	t = tcg_temp_new_i32();
3212	pd = tcg_temp_new_ptr();
3213	zn = tcg_temp_new_ptr();
3214	pg = tcg_temp_new_ptr();
3215
3216	tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd));
3217	tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn));
3218	tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg));
3219
3220	gen_fn(t, pd, zn, pg, tcg_constant_i32(simd_desc(vsz, vsz, a->imm)));
3221
3222	do_pred_flags(t);
3223	return true;
3224	}
3225
3226	#define DO_PPZI(NAME, name) \
3227	static gen_helper_gvec_flags_3 * const name##_ppzi_fns[4] = { \
3228	gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3229	gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3230	}; \
3231	TRANS_FEAT(NAME##_ppzi, aa64_sme_or_sve, do_ppzi_flags, a, \
3232	name##_ppzi_fns[a->esz])
3233
3234	DO_PPZI(CMPEQ, cmpeq)
3235	DO_PPZI(CMPNE, cmpne)
3236	DO_PPZI(CMPGT, cmpgt)
3237	DO_PPZI(CMPGE, cmpge)
3238	DO_PPZI(CMPHI, cmphi)
3239	DO_PPZI(CMPHS, cmphs)
3240	DO_PPZI(CMPLT, cmplt)
3241	DO_PPZI(CMPLE, cmple)
3242	DO_PPZI(CMPLO, cmplo)
3243	DO_PPZI(CMPLS, cmpls)
3244
3245	#undef DO_PPZI
3246
3247	/*
3248	*** SVE Partition Break Group
3249	*/
3250
3251	static bool do_brk3(DisasContext s, arg_rprr_s a,
3252	gen_helper_gvec_4 fn, gen_helper_gvec_flags_4 fn_s)
3253	{
3254	if (!sve_access_check(s)) {
3255	return true;
3256	}
3257
3258	unsigned vsz = pred_full_reg_size(s);
3259
3260	/* Predicate sizes may be smaller and cannot use simd_desc. */
3261	TCGv_ptr d = tcg_temp_new_ptr();
3262	TCGv_ptr n = tcg_temp_new_ptr();
3263	TCGv_ptr m = tcg_temp_new_ptr();
3264	TCGv_ptr g = tcg_temp_new_ptr();
3265	TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz));
3266
3267	tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd));
3268	tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn));
3269	tcg_gen_addi_ptr(m, tcg_env, pred_full_reg_offset(s, a->rm));
3270	tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg));
3271
3272	if (a->s) {
3273	TCGv_i32 t = tcg_temp_new_i32();
3274	fn_s(t, d, n, m, g, desc);
3275	do_pred_flags(t);
3276	} else {
3277	fn(d, n, m, g, desc);
3278	}
3279	return true;
3280	}
3281
3282	static bool do_brk2(DisasContext s, arg_rpr_s a,
3283	gen_helper_gvec_3 fn, gen_helper_gvec_flags_3 fn_s)
3284	{
3285	if (!sve_access_check(s)) {
3286	return true;
3287	}
3288
3289	unsigned vsz = pred_full_reg_size(s);
3290
3291	/* Predicate sizes may be smaller and cannot use simd_desc. */
3292	TCGv_ptr d = tcg_temp_new_ptr();
3293	TCGv_ptr n = tcg_temp_new_ptr();
3294	TCGv_ptr g = tcg_temp_new_ptr();
3295	TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz));
3296
3297	tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd));
3298	tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn));
3299	tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg));
3300
3301	if (a->s) {
3302	TCGv_i32 t = tcg_temp_new_i32();
3303	fn_s(t, d, n, g, desc);
3304	do_pred_flags(t);
3305	} else {
3306	fn(d, n, g, desc);
3307	}
3308	return true;
3309	}
3310
3311	TRANS_FEAT(BRKPA, aa64_sme_or_sve, do_brk3, a,
3312	gen_helper_sve_brkpa, gen_helper_sve_brkpas)
3313	TRANS_FEAT(BRKPB, aa64_sme_or_sve, do_brk3, a,
3314	gen_helper_sve_brkpb, gen_helper_sve_brkpbs)
3315
3316	TRANS_FEAT(BRKA_m, aa64_sme_or_sve, do_brk2, a,
3317	gen_helper_sve_brka_m, gen_helper_sve_brkas_m)
3318	TRANS_FEAT(BRKB_m, aa64_sme_or_sve, do_brk2, a,
3319	gen_helper_sve_brkb_m, gen_helper_sve_brkbs_m)
3320
3321	TRANS_FEAT(BRKA_z, aa64_sme_or_sve, do_brk2, a,
3322	gen_helper_sve_brka_z, gen_helper_sve_brkas_z)
3323	TRANS_FEAT(BRKB_z, aa64_sme_or_sve, do_brk2, a,
3324	gen_helper_sve_brkb_z, gen_helper_sve_brkbs_z)
3325
3326	TRANS_FEAT(BRKN, aa64_sme_or_sve, do_brk2, a,
3327	gen_helper_sve_brkn, gen_helper_sve_brkns)
3328
3329	/*
3330	*** SVE Predicate Count Group
3331	*/
3332
3333	static void do_cntp(DisasContext *s, TCGv_i64 val, int esz, int pn, int pg)
3334	{
3335	unsigned psz = pred_full_reg_size(s);
3336
3337	if (psz <= 8) {
3338	uint64_t psz_mask;
3339
3340	tcg_gen_ld_i64(val, tcg_env, pred_full_reg_offset(s, pn));
3341	if (pn != pg) {
3342	TCGv_i64 g = tcg_temp_new_i64();
3343	tcg_gen_ld_i64(g, tcg_env, pred_full_reg_offset(s, pg));
3344	tcg_gen_and_i64(val, val, g);
3345	}
3346
3347	/* Reduce the pred_esz_masks value simply to reduce the
3348	* size of the code generated here.
3349	*/
3350	psz_mask = MAKE_64BIT_MASK(0, psz * 8);
3351	tcg_gen_andi_i64(val, val, pred_esz_masks[esz] & psz_mask);
3352
3353	tcg_gen_ctpop_i64(val, val);
3354	} else {
3355	TCGv_ptr t_pn = tcg_temp_new_ptr();
3356	TCGv_ptr t_pg = tcg_temp_new_ptr();
3357	unsigned desc = 0;
3358
3359	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, psz);
3360	desc = FIELD_DP32(desc, PREDDESC, ESZ, esz);
3361
3362	tcg_gen_addi_ptr(t_pn, tcg_env, pred_full_reg_offset(s, pn));
3363	tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
3364
3365	gen_helper_sve_cntp(val, t_pn, t_pg, tcg_constant_i32(desc));
3366	}
3367	}
3368
3369	static bool trans_CNTP(DisasContext s, arg_CNTP a)
3370	{
3371	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3372	return false;
3373	}
3374	if (sve_access_check(s)) {
3375	do_cntp(s, cpu_reg(s, a->rd), a->esz, a->rn, a->pg);
3376	}
3377	return true;
3378	}
3379
3380	static bool trans_CNTP_c(DisasContext s, arg_CNTP_c a)
3381	{
3382	TCGv_i32 t_png;
3383	uint32_t desc = 0;
3384
3385	if (dc_isar_feature(aa64_sve2p1, s)) {
3386	if (!sve_access_check(s)) {
3387	return true;
3388	}
3389	} else if (dc_isar_feature(aa64_sme2, s)) {
3390	if (!sme_sm_enabled_check(s)) {
3391	return true;
3392	}
3393	} else {
3394	return false;
3395	}
3396
3397	t_png = tcg_temp_new_i32();
3398	tcg_gen_ld16u_i32(t_png, tcg_env,
3399	pred_full_reg_offset(s, a->rn) ^
3400	(HOST_BIG_ENDIAN ? 6 : 0));
3401
3402	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
3403	desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3404	desc = FIELD_DP32(desc, PREDDESC, DATA, a->vl);
3405
3406	gen_helper_sve2p1_cntp_c(cpu_reg(s, a->rd), t_png, tcg_constant_i32(desc));
3407	return true;
3408	}
3409
3410	static bool trans_INCDECP_r(DisasContext s, arg_incdec_pred a)
3411	{
3412	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3413	return false;
3414	}
3415	if (sve_access_check(s)) {
3416	TCGv_i64 reg = cpu_reg(s, a->rd);
3417	TCGv_i64 val = tcg_temp_new_i64();
3418
3419	do_cntp(s, val, a->esz, a->pg, a->pg);
3420	if (a->d) {
3421	tcg_gen_sub_i64(reg, reg, val);
3422	} else {
3423	tcg_gen_add_i64(reg, reg, val);
3424	}
3425	}
3426	return true;
3427	}
3428
3429	static bool trans_INCDECP_z(DisasContext s, arg_incdec2_pred a)
3430	{
3431	if (a->esz == 0 \|\| !dc_isar_feature(aa64_sme_or_sve, s)) {
3432	return false;
3433	}
3434	if (sve_access_check(s)) {
3435	unsigned vsz = vec_full_reg_size(s);
3436	TCGv_i64 val = tcg_temp_new_i64();
3437	GVecGen2sFn *gvec_fn = a->d ? tcg_gen_gvec_subs : tcg_gen_gvec_adds;
3438
3439	do_cntp(s, val, a->esz, a->pg, a->pg);
3440	gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
3441	vec_full_reg_offset(s, a->rn), val, vsz, vsz);
3442	}
3443	return true;
3444	}
3445
3446	static bool trans_SINCDECP_r_32(DisasContext s, arg_incdec_pred a)
3447	{
3448	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3449	return false;
3450	}
3451	if (sve_access_check(s)) {
3452	TCGv_i64 reg = cpu_reg(s, a->rd);
3453	TCGv_i64 val = tcg_temp_new_i64();
3454
3455	do_cntp(s, val, a->esz, a->pg, a->pg);
3456	do_sat_addsub_32(reg, val, a->u, a->d);
3457	}
3458	return true;
3459	}
3460
3461	static bool trans_SINCDECP_r_64(DisasContext s, arg_incdec_pred a)
3462	{
3463	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3464	return false;
3465	}
3466	if (sve_access_check(s)) {
3467	TCGv_i64 reg = cpu_reg(s, a->rd);
3468	TCGv_i64 val = tcg_temp_new_i64();
3469
3470	do_cntp(s, val, a->esz, a->pg, a->pg);
3471	do_sat_addsub_64(reg, val, a->u, a->d);
3472	}
3473	return true;
3474	}
3475
3476	static bool trans_SINCDECP_z(DisasContext s, arg_incdec2_pred a)
3477	{
3478	if (a->esz == 0 \|\| !dc_isar_feature(aa64_sme_or_sve, s)) {
3479	return false;
3480	}
3481	if (sve_access_check(s)) {
3482	TCGv_i64 val = tcg_temp_new_i64();
3483	do_cntp(s, val, a->esz, a->pg, a->pg);
3484	do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, a->u, a->d);
3485	}
3486	return true;
3487	}
3488
3489	static bool do_firstp_lastp(DisasContext s, arg_rpr_esz a, bool firstp)
3490	{
3491	if (sve_access_check(s)) {
3492	unsigned psz = pred_full_reg_size(s);
3493	TCGv_i64 v = cpu_reg(s, a->rd);
3494
3495	if (psz <= 8) {
3496	uint64_t psz_mask;
3497
3498	tcg_gen_ld_i64(v, tcg_env, pred_full_reg_offset(s, a->rn));
3499	if (a->rn != a->pg) {
3500	TCGv_i64 g = tcg_temp_new_i64();
3501	tcg_gen_ld_i64(g, tcg_env, pred_full_reg_offset(s, a->pg));
3502	tcg_gen_and_i64(v, v, g);
3503	}
3504
3505	/*
3506	* Reduce the pred_esz_masks value simply to reduce the
3507	* size of the code generated here.
3508	*/
3509	psz_mask = MAKE_64BIT_MASK(0, psz * 8);
3510	tcg_gen_andi_i64(v, v, pred_esz_masks[a->esz] & psz_mask);
3511
3512	if (firstp) {
3513	tcg_gen_ctzi_i64(v, v, -1);
3514	} else {
3515	tcg_gen_clzi_i64(v, v, 64);
3516	tcg_gen_subfi_i64(v, 63, v);
3517	}
3518	tcg_gen_sari_i64(v, v, a->esz);
3519	} else {
3520	TCGv_ptr t_pn = tcg_temp_new_ptr();
3521	TCGv_ptr t_pg = tcg_temp_new_ptr();
3522	unsigned desc = 0;
3523	TCGv_i32 t_desc;
3524
3525	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, psz);
3526	desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3527
3528	tcg_gen_addi_ptr(t_pn, tcg_env, pred_full_reg_offset(s, a->rn));
3529	tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg));
3530	t_desc = tcg_constant_i32(desc);
3531
3532	if (firstp) {
3533	gen_helper_sve_firstp(v, t_pn, t_pg, t_desc);
3534	} else {
3535	gen_helper_sve_lastp(v, t_pn, t_pg, t_desc);
3536	}
3537	}
3538	}
3539	return true;
3540	}
3541
3542	TRANS_FEAT(FIRSTP, aa64_sme2p2_or_sve2p2, do_firstp_lastp, a, true)
3543	TRANS_FEAT(LASTP, aa64_sme2p2_or_sve2p2, do_firstp_lastp, a, false)
3544
3545	/*
3546	*** SVE Integer Compare Scalars Group
3547	*/
3548
3549	static bool trans_CTERM(DisasContext s, arg_CTERM a)
3550	{
3551	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3552	return false;
3553	}
3554	if (!sve_access_check(s)) {
3555	return true;
3556	}
3557
3558	TCGCond cond = (a->ne ? TCG_COND_NE : TCG_COND_EQ);
3559	TCGv_i64 rn = read_cpu_reg(s, a->rn, a->sf);
3560	TCGv_i64 rm = read_cpu_reg(s, a->rm, a->sf);
3561	TCGv_i64 cmp = tcg_temp_new_i64();
3562
3563	tcg_gen_setcond_i64(cond, cmp, rn, rm);
3564	tcg_gen_extrl_i64_i32(cpu_NF, cmp);
3565
3566	/* VF = !NF & !CF. */
3567	tcg_gen_xori_i32(cpu_VF, cpu_NF, 1);
3568	tcg_gen_andc_i32(cpu_VF, cpu_VF, cpu_CF);
3569
3570	/* Both NF and VF actually look at bit 31. */
3571	tcg_gen_neg_i32(cpu_NF, cpu_NF);
3572	tcg_gen_neg_i32(cpu_VF, cpu_VF);
3573	return true;
3574	}
3575
3576	typedef void gen_while_fn(TCGv_i32, TCGv_ptr, TCGv_i32, TCGv_i32);
3577	static bool do_WHILE(DisasContext s, arg_while a,
3578	bool lt, int scale, int data, gen_while_fn *fn)
3579	{
3580	TCGv_i64 op0, op1, t0, t1, tmax;
3581	TCGv_i32 t2;
3582	TCGv_ptr ptr;
3583	unsigned vsz = vec_full_reg_size(s);
3584	unsigned desc = 0;
3585	TCGCond cond;
3586	uint64_t maxval;
3587	/* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3588	bool eq = a->eq == lt;
3589
3590	if (!sve_access_check(s)) {
3591	return true;
3592	}
3593
3594	op0 = read_cpu_reg(s, a->rn, 1);
3595	op1 = read_cpu_reg(s, a->rm, 1);
3596
3597	if (!a->sf) {
3598	if (a->u) {
3599	tcg_gen_ext32u_i64(op0, op0);
3600	tcg_gen_ext32u_i64(op1, op1);
3601	} else {
3602	tcg_gen_ext32s_i64(op0, op0);
3603	tcg_gen_ext32s_i64(op1, op1);
3604	}
3605	}
3606
3607	/* For the helper, compress the different conditions into a computation
3608	* of how many iterations for which the condition is true.
3609	*/
3610	t0 = tcg_temp_new_i64();
3611	t1 = tcg_temp_new_i64();
3612
3613	if (lt) {
3614	tcg_gen_sub_i64(t0, op1, op0);
3615	if (a->u) {
3616	maxval = a->sf ? UINT64_MAX : UINT32_MAX;
3617	cond = eq ? TCG_COND_LEU : TCG_COND_LTU;
3618	} else {
3619	maxval = a->sf ? INT64_MAX : INT32_MAX;
3620	cond = eq ? TCG_COND_LE : TCG_COND_LT;
3621	}
3622	} else {
3623	tcg_gen_sub_i64(t0, op0, op1);
3624	if (a->u) {
3625	maxval = 0;
3626	cond = eq ? TCG_COND_GEU : TCG_COND_GTU;
3627	} else {
3628	maxval = a->sf ? INT64_MIN : INT32_MIN;
3629	cond = eq ? TCG_COND_GE : TCG_COND_GT;
3630	}
3631	}
3632
3633	tmax = tcg_constant_i64((vsz << scale) >> a->esz);
3634	if (eq) {
3635	/* Equality means one more iteration. */
3636	tcg_gen_addi_i64(t0, t0, 1);
3637
3638	/*
3639	* For the less-than while, if op1 is maxval (and the only time
3640	* the addition above could overflow), then we produce an all-true
3641	* predicate by setting the count to the vector length. This is
3642	* because the pseudocode is described as an increment + compare
3643	* loop, and the maximum integer would always compare true.
3644	* Similarly, the greater-than while has the same issue with the
3645	* minimum integer due to the decrement + compare loop.
3646	*/
3647	tcg_gen_movi_i64(t1, maxval);
3648	tcg_gen_movcond_i64(TCG_COND_EQ, t0, op1, t1, tmax, t0);
3649	}
3650
3651	/* Bound to the maximum. */
3652	tcg_gen_umin_i64(t0, t0, tmax);
3653
3654	/* Set the count to zero if the condition is false. */
3655	tcg_gen_movi_i64(t1, 0);
3656	tcg_gen_movcond_i64(cond, t0, op0, op1, t0, t1);
3657
3658	/* Since we're bounded, pass as a 32-bit type. */
3659	t2 = tcg_temp_new_i32();
3660	tcg_gen_extrl_i64_i32(t2, t0);
3661
3662	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8);
3663	desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3664	desc = FIELD_DP32(desc, PREDDESC, DATA, data);
3665
3666	ptr = tcg_temp_new_ptr();
3667	tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd));
3668
3669	fn(t2, ptr, t2, tcg_constant_i32(desc));
3670
3671	do_pred_flags(t2);
3672	return true;
3673	}
3674
3675	TRANS_FEAT(WHILE_lt, aa64_sme_or_sve, do_WHILE,
3676	a, true, 0, 0, gen_helper_sve_whilel)
3677	TRANS_FEAT(WHILE_gt, aa64_sme_or_sve2, do_WHILE,
3678	a, false, 0, 0, gen_helper_sve_whileg)
3679
3680	TRANS_FEAT(WHILE_lt_pair, aa64_sme2_or_sve2p1, do_WHILE,
3681	a, true, 1, 0, gen_helper_sve_while2l)
3682	TRANS_FEAT(WHILE_gt_pair, aa64_sme2_or_sve2p1, do_WHILE,
3683	a, false, 1, 0, gen_helper_sve_while2g)
3684
3685	TRANS_FEAT(WHILE_lt_cnt2, aa64_sme2_or_sve2p1, do_WHILE,
3686	a, true, 1, 1, gen_helper_sve_whilecl)
3687	TRANS_FEAT(WHILE_lt_cnt4, aa64_sme2_or_sve2p1, do_WHILE,
3688	a, true, 2, 2, gen_helper_sve_whilecl)
3689	TRANS_FEAT(WHILE_gt_cnt2, aa64_sme2_or_sve2p1, do_WHILE,
3690	a, false, 1, 1, gen_helper_sve_whilecg)
3691	TRANS_FEAT(WHILE_gt_cnt4, aa64_sme2_or_sve2p1, do_WHILE,
3692	a, false, 2, 2, gen_helper_sve_whilecg)
3693
3694	static bool trans_WHILE_ptr(DisasContext s, arg_WHILE_ptr a)
3695	{
3696	TCGv_i64 op0, op1, diff, t1;
3697	TCGv_i32 t2;
3698	TCGv_ptr ptr;
3699	unsigned vsz = vec_full_reg_size(s);
3700	unsigned desc = 0;
3701
3702	if (!dc_isar_feature(aa64_sme_or_sve2, s)) {
3703	return false;
3704	}
3705	if (!sve_access_check(s)) {
3706	return true;
3707	}
3708
3709	op0 = read_cpu_reg(s, a->rn, 1);
3710	op1 = read_cpu_reg(s, a->rm, 1);
3711
3712	diff = tcg_temp_new_i64();
3713
3714	if (a->rw) {
3715	/* WHILERW */
3716	/* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3717	t1 = tcg_temp_new_i64();
3718	tcg_gen_sub_i64(diff, op0, op1);
3719	tcg_gen_sub_i64(t1, op1, op0);
3720	tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, diff, t1);
3721	} else {
3722	/* WHILEWR */
3723	/* Saturating subtraction maps diff <= 0 to diff == 0. */
3724	tcg_gen_ussub_i64(diff, op1, op0);
3725	}
3726
3727	/* Divide, rounding down, by ESIZE. */
3728	tcg_gen_shri_i64(diff, diff, a->esz);
3729
3730	/*
3731	* If diff == 0, the condition is always true. Also, bound to max.
3732	* Simplify
3733	* diff = diff ? diff : max;
3734	* diff = umin(diff, max);
3735	* via
3736	* diff -= 1;
3737	* diff = umin(diff, max - 1);
3738	* diff += 1;
3739	* via 0 - 1 == UINT64_MAX.
3740	*/
3741	tcg_gen_addi_i64(diff, diff, -1);
3742	tcg_gen_umin_i64(diff, diff, tcg_constant_i64((vsz >> a->esz) - 1));
3743
3744	/*
3745	* Since we're bounded, pass as a 32-bit type.
3746	* Sink the diff += 1 from above into the 32-bit type.
3747	*/
3748	t2 = tcg_temp_new_i32();
3749	tcg_gen_extrl_i64_i32(t2, diff);
3750	tcg_gen_addi_i32(t2, t2, 1);
3751
3752	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8);
3753	desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3754
3755	ptr = tcg_temp_new_ptr();
3756	tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd));
3757
3758	gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc));
3759	do_pred_flags(t2);
3760	return true;
3761	}
3762
3763	static bool do_pext(DisasContext s, arg_pext a, int n)
3764	{
3765	TCGv_i32 t_png;
3766	TCGv_ptr t_pd;
3767	int pl;
3768
3769	if (!sve_access_check(s)) {
3770	return true;
3771	}
3772
3773	t_png = tcg_temp_new_i32();
3774	tcg_gen_ld16u_i32(t_png, tcg_env,
3775	pred_full_reg_offset(s, a->rn) ^
3776	(HOST_BIG_ENDIAN ? 6 : 0));
3777
3778	t_pd = tcg_temp_new_ptr();
3779	pl = pred_full_reg_size(s);
3780
3781	for (int i = 0; i < n; ++i) {
3782	int rd = (a->rd + i) % 16;
3783	int part = a->imm * n + i;
3784	unsigned desc = 0;
3785
3786	desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pl);
3787	desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3788	desc = FIELD_DP32(desc, PREDDESC, DATA, part);
3789
3790	tcg_gen_addi_ptr(t_pd, tcg_env, pred_full_reg_offset(s, rd));
3791	gen_helper_pext(t_pd, t_png, tcg_constant_i32(desc));
3792	}
3793	return true;
3794	}
3795
3796	TRANS_FEAT(PEXT_1, aa64_sme2_or_sve2p1, do_pext, a, 1)
3797	TRANS_FEAT(PEXT_2, aa64_sme2_or_sve2p1, do_pext, a, 2)
3798
3799	/*
3800	*** SVE Integer Wide Immediate - Unpredicated Group
3801	*/
3802
3803	static bool trans_FDUP(DisasContext s, arg_FDUP a)
3804	{
3805	if (a->esz == 0 \|\| !dc_isar_feature(aa64_sme_or_sve, s)) {
3806	return false;
3807	}
3808	if (sve_access_check(s)) {
3809	unsigned vsz = vec_full_reg_size(s);
3810	int dofs = vec_full_reg_offset(s, a->rd);
3811	uint64_t imm;
3812
3813	/* Decode the VFP immediate. */
3814	imm = vfp_expand_imm(a->esz, a->imm);
3815	tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, imm);
3816	}
3817	return true;
3818	}
3819
3820	static bool trans_DUP_i(DisasContext s, arg_DUP_i a)
3821	{
3822	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3823	return false;
3824	}
3825	if (sve_access_check(s)) {
3826	unsigned vsz = vec_full_reg_size(s);
3827	int dofs = vec_full_reg_offset(s, a->rd);
3828	tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, a->imm);
3829	}
3830	return true;
3831	}
3832
3833	TRANS_FEAT(ADD_zzi, aa64_sme_or_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_addi, a)
3834
3835	static bool trans_SUB_zzi(DisasContext s, arg_rri_esz a)
3836	{
3837	a->imm = -a->imm;
3838	return trans_ADD_zzi(s, a);
3839	}
3840
3841	static bool trans_SUBR_zzi(DisasContext s, arg_rri_esz a)
3842	{
3843	static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 };
3844	static const GVecGen2s op[4] = {
3845	{ .fni8 = tcg_gen_vec_sub8_i64,
3846	.fniv = tcg_gen_sub_vec,
3847	.fno = gen_helper_sve_subri_b,
3848	.opt_opc = vecop_list,
3849	.vece = MO_8,
3850	.scalar_first = true },
3851	{ .fni8 = tcg_gen_vec_sub16_i64,
3852	.fniv = tcg_gen_sub_vec,
3853	.fno = gen_helper_sve_subri_h,
3854	.opt_opc = vecop_list,
3855	.vece = MO_16,
3856	.scalar_first = true },
3857	{ .fni4 = tcg_gen_sub_i32,
3858	.fniv = tcg_gen_sub_vec,
3859	.fno = gen_helper_sve_subri_s,
3860	.opt_opc = vecop_list,
3861	.vece = MO_32,
3862	.scalar_first = true },
3863	{ .fni8 = tcg_gen_sub_i64,
3864	.fniv = tcg_gen_sub_vec,
3865	.fno = gen_helper_sve_subri_d,
3866	.opt_opc = vecop_list,
3867	.prefer_i64 = true,
3868	.vece = MO_64,
3869	.scalar_first = true }
3870	};
3871
3872	if (!dc_isar_feature(aa64_sme_or_sve, s)) {
3873	return false;
3874	}
3875	if (sve_access_check(s)) {
3876	unsigned vsz = vec_full_reg_size(s);
3877	tcg_gen_gvec_2s(vec_full_reg_offset(s, a->rd),
3878	vec_full_reg_offset(s, a->rn),
3879	vsz, vsz, tcg_constant_i64(a->imm), &op[a->esz]);
3880	}
3881	return true;
3882	}
3883
3884	TRANS_FEAT(MUL_zzi, aa64_sme_or_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_muli, a)
3885
3886	static bool do_zzi_sat(DisasContext s, arg_rri_esz a, bool u, bool d)
3887	{
3888	if (sve_access_check(s)) {
3889	do_sat_addsub_vec(s, a->esz, a->rd, a->rn,
3890	tcg_constant_i64(a->imm), u, d);
3891	}
3892	return true;
3893	}
3894
3895	TRANS_FEAT(SQADD_zzi, aa64_sme_or_sve, do_zzi_sat, a, false, false)
3896	TRANS_FEAT(UQADD_zzi, aa64_sme_or_sve, do_zzi_sat, a, true, false)
3897	TRANS_FEAT(SQSUB_zzi, aa64_sme_or_sve, do_zzi_sat, a, false, true)
3898	TRANS_FEAT(UQSUB_zzi, aa64_sme_or_sve, do_zzi_sat, a, true, true)
3899
3900	static bool do_zzi_ool(DisasContext s, arg_rri_esz a, gen_helper_gvec_2i *fn)
3901	{
3902	if (sve_access_check(s)) {
3903	unsigned vsz = vec_full_reg_size(s);
3904	tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd),
3905	vec_full_reg_offset(s, a->rn),
3906	tcg_constant_i64(a->imm), vsz, vsz, 0, fn);
3907	}
3908	return true;
3909	}
3910
3911	#define DO_ZZI(NAME, name) \
3912	static gen_helper_gvec_2i * const name##i_fns[4] = { \
3913	gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3914	gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3915	}; \
3916	TRANS_FEAT(NAME##_zzi, aa64_sme_or_sve, do_zzi_ool, a, name##i_fns[a->esz])
3917
3918	DO_ZZI(SMAX, smax)
3919	DO_ZZI(UMAX, umax)
3920	DO_ZZI(SMIN, smin)
3921	DO_ZZI(UMIN, umin)
3922
3923	#undef DO_ZZI
3924
3925	static gen_helper_gvec_4 * const dot_fns[2][2] = {
3926	{ gen_helper_gvec_sdot_4b, gen_helper_gvec_sdot_4h },
3927	{ gen_helper_gvec_udot_4b, gen_helper_gvec_udot_4h }
3928	};
3929	TRANS_FEAT(DOT_zzzz, aa64_sme_or_sve, gen_gvec_ool_zzzz,
3930	dot_fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0)
3931
3932	/*
3933	* SVE Multiply - Indexed
3934	*/
3935
3936	TRANS_FEAT(SDOT_zzxw_4s, aa64_sme_or_sve, gen_gvec_ool_arg_zzxz,
3937	gen_helper_gvec_sdot_idx_4b, a)
3938	TRANS_FEAT(SDOT_zzxw_4d, aa64_sme_or_sve, gen_gvec_ool_arg_zzxz,
3939	gen_helper_gvec_sdot_idx_4h, a)
3940	TRANS_FEAT(UDOT_zzxw_4s, aa64_sme_or_sve, gen_gvec_ool_arg_zzxz,
3941	gen_helper_gvec_udot_idx_4b, a)
3942	TRANS_FEAT(UDOT_zzxw_4d, aa64_sme_or_sve, gen_gvec_ool_arg_zzxz,
3943	gen_helper_gvec_udot_idx_4h, a)
3944
3945	TRANS_FEAT(SUDOT_zzxw_4s, aa64_sme_sve_i8mm, gen_gvec_ool_arg_zzxz,
3946	gen_helper_gvec_sudot_idx_4b, a)
3947	TRANS_FEAT(USDOT_zzxw_4s, aa64_sme_sve_i8mm, gen_gvec_ool_arg_zzxz,
3948	gen_helper_gvec_usdot_idx_4b, a)
3949
3950	TRANS_FEAT(SDOT_zzxw_2s, aa64_sme2_or_sve2p1, gen_gvec_ool_arg_zzxz,
3951	gen_helper_gvec_sdot_idx_2h, a)
3952	TRANS_FEAT(UDOT_zzxw_2s, aa64_sme2_or_sve2p1, gen_gvec_ool_arg_zzxz,
3953	gen_helper_gvec_udot_idx_2h, a)
3954
3955	#define DO_SVE2_RRX(NAME, FUNC) \
3956	TRANS_FEAT(NAME, aa64_sme_or_sve2, gen_gvec_ool_zzz, FUNC, \
3957	a->rd, a->rn, a->rm, a->index)
3958
3959	DO_SVE2_RRX(MUL_zzx_h, gen_helper_gvec_mul_idx_h)
3960	DO_SVE2_RRX(MUL_zzx_s, gen_helper_gvec_mul_idx_s)
3961	DO_SVE2_RRX(MUL_zzx_d, gen_helper_gvec_mul_idx_d)
3962
3963	DO_SVE2_RRX(SQDMULH_zzx_h, gen_helper_sve2_sqdmulh_idx_h)
3964	DO_SVE2_RRX(SQDMULH_zzx_s, gen_helper_sve2_sqdmulh_idx_s)
3965	DO_SVE2_RRX(SQDMULH_zzx_d, gen_helper_sve2_sqdmulh_idx_d)
3966
3967	DO_SVE2_RRX(SQRDMULH_zzx_h, gen_helper_sve2_sqrdmulh_idx_h)
3968	DO_SVE2_RRX(SQRDMULH_zzx_s, gen_helper_sve2_sqrdmulh_idx_s)
3969	DO_SVE2_RRX(SQRDMULH_zzx_d, gen_helper_sve2_sqrdmulh_idx_d)
3970
3971	#undef DO_SVE2_RRX
3972
3973	#define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3974	TRANS_FEAT(NAME, aa64_sme_or_sve2, gen_gvec_ool_zzz, FUNC, \
3975	a->rd, a->rn, a->rm, (a->index << 1) \| TOP)
3976
3977	DO_SVE2_RRX_TB(SQDMULLB_zzx_s, gen_helper_sve2_sqdmull_idx_s, false)
3978	DO_SVE2_RRX_TB(SQDMULLB_zzx_d, gen_helper_sve2_sqdmull_idx_d, false)
3979	DO_SVE2_RRX_TB(SQDMULLT_zzx_s, gen_helper_sve2_sqdmull_idx_s, true)
3980	DO_SVE2_RRX_TB(SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true)
3981
3982	DO_SVE2_RRX_TB(SMULLB_zzx_s, gen_helper_sve2_smull_idx_s, false)
3983	DO_SVE2_RRX_TB(SMULLB_zzx_d, gen_helper_sve2_smull_idx_d, false)
3984	DO_SVE2_RRX_TB(SMULLT_zzx_s, gen_helper_sve2_smull_idx_s, true)
3985	DO_SVE2_RRX_TB(SMULLT_zzx_d, gen_helper_sve2_smull_idx_d, true)
3986
3987	DO_SVE2_RRX_TB(UMULLB_zzx_s, gen_helper_sve2_umull_idx_s, false)
3988	DO_SVE2_RRX_TB(UMULLB_zzx_d, gen_helper_sve2_umull_idx_d, false)
3989	DO_SVE2_RRX_TB(UMULLT_zzx_s, gen_helper_sve2_umull_idx_s, true)
3990	DO_SVE2_RRX_TB(UMULLT_zzx_d, gen_helper_sve2_umull_idx_d, true)
3991
3992	#undef DO_SVE2_RRX_TB
3993
3994	#define DO_SVE2_RRXR(NAME, FUNC) \
3995	TRANS_FEAT(NAME, aa64_sme_or_sve2, gen_gvec_ool_arg_zzxz, FUNC, a)
3996
3997	DO_SVE2_RRXR(MLA_zzxz_h, gen_helper_gvec_mla_idx_h)
3998	DO_SVE2_RRXR(MLA_zzxz_s, gen_helper_gvec_mla_idx_s)
3999	DO_SVE2_RRXR(MLA_zzxz_d, gen_helper_gvec_mla_idx_d)
4000
4001	DO_SVE2_RRXR(MLS_zzxz_h, gen_helper_gvec_mls_idx_h)
4002	DO_SVE2_RRXR(MLS_zzxz_s, gen_helper_gvec_mls_idx_s)
4003	DO_SVE2_RRXR(MLS_zzxz_d, gen_helper_gvec_mls_idx_d)
4004
4005	DO_SVE2_RRXR(SQRDMLAH_zzxz_h, gen_helper_sve2_sqrdmlah_idx_h)
4006	DO_SVE2_RRXR(SQRDMLAH_zzxz_s, gen_helper_sve2_sqrdmlah_idx_s)
4007	DO_SVE2_RRXR(SQRDMLAH_zzxz_d, gen_helper_sve2_sqrdmlah_idx_d)
4008
4009	DO_SVE2_RRXR(SQRDMLSH_zzxz_h, gen_helper_sve2_sqrdmlsh_idx_h)
4010	DO_SVE2_RRXR(SQRDMLSH_zzxz_s, gen_helper_sve2_sqrdmlsh_idx_s)
4011	DO_SVE2_RRXR(SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d)
4012
4013	#undef DO_SVE2_RRXR
4014
4015	#define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
4016	TRANS_FEAT(NAME, aa64_sme_or_sve2, gen_gvec_ool_zzzz, FUNC, \
4017	a->rd, a->rn, a->rm, a->ra, (a->index << 1) \| TOP)
4018
4019	DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false)
4020	DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false)
4021	DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, true)
4022	DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, true)
4023
4024	DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, false)
4025	DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false)
4026	DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true)
4027	DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true)
4028
4029	DO_SVE2_RRXR_TB(SMLALB_zzxw_s, gen_helper_sve2_smlal_idx_s, false)
4030	DO_SVE2_RRXR_TB(SMLALB_zzxw_d, gen_helper_sve2_smlal_idx_d, false)
4031	DO_SVE2_RRXR_TB(SMLALT_zzxw_s, gen_helper_sve2_smlal_idx_s, true)
4032	DO_SVE2_RRXR_TB(SMLALT_zzxw_d, gen_helper_sve2_smlal_idx_d, true)
4033
4034	DO_SVE2_RRXR_TB(UMLALB_zzxw_s, gen_helper_sve2_umlal_idx_s, false)
4035	DO_SVE2_RRXR_TB(UMLALB_zzxw_d, gen_helper_sve2_umlal_idx_d, false)
4036	DO_SVE2_RRXR_TB(UMLALT_zzxw_s, gen_helper_sve2_umlal_idx_s, true)
4037	DO_SVE2_RRXR_TB(UMLALT_zzxw_d, gen_helper_sve2_umlal_idx_d, true)
4038
4039	DO_SVE2_RRXR_TB(SMLSLB_zzxw_s, gen_helper_sve2_smlsl_idx_s, false)
4040	DO_SVE2_RRXR_TB(SMLSLB_zzxw_d, gen_helper_sve2_smlsl_idx_d, false)
4041	DO_SVE2_RRXR_TB(SMLSLT_zzxw_s, gen_helper_sve2_smlsl_idx_s, true)
4042	DO_SVE2_RRXR_TB(SMLSLT_zzxw_d, gen_helper_sve2_smlsl_idx_d, true)
4043
4044	DO_SVE2_RRXR_TB(UMLSLB_zzxw_s, gen_helper_sve2_umlsl_idx_s, false)
4045	DO_SVE2_RRXR_TB(UMLSLB_zzxw_d, gen_helper_sve2_umlsl_idx_d, false)
4046	DO_SVE2_RRXR_TB(UMLSLT_zzxw_s, gen_helper_sve2_umlsl_idx_s, true)
4047	DO_SVE2_RRXR_TB(UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true)
4048
4049	#undef DO_SVE2_RRXR_TB
4050
4051	#define DO_SVE2_RRXR_ROT(NAME, FUNC) \
4052	TRANS_FEAT(NAME, aa64_sme_or_sve2, gen_gvec_ool_zzzz, FUNC, \
4053	a->rd, a->rn, a->rm, a->ra, (a->index << 2) \| a->rot)
4054
4055	DO_SVE2_RRXR_ROT(CMLA_zzxz_h, gen_helper_sve2_cmla_idx_h)
4056	DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s)
4057
4058	DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h)
4059	DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s)
4060
4061	DO_SVE2_RRXR_ROT(CDOT_zzxw_s, gen_helper_sve2_cdot_idx_s)
4062	DO_SVE2_RRXR_ROT(CDOT_zzxw_d, gen_helper_sve2_cdot_idx_d)
4063
4064	#undef DO_SVE2_RRXR_ROT
4065
4066	/*
4067	*** SVE Floating Point Multiply-Add Indexed Group
4068	*/
4069
4070	static bool do_fmla_zzxz(DisasContext s, arg_rrxr_esz a,
4071	gen_helper_gvec_4_ptr *fn)
4072	{
4073	/* These insns use MO_8 to encode BFloat16 */
4074	if (a->esz == MO_8 && !dc_isar_feature(aa64_sve_b16b16, s)) {
4075	return false;
4076	}
4077	return gen_gvec_fpst_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index,
4078	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4079	}
4080
4081	static gen_helper_gvec_4_ptr * const fmla_idx_fns[4] = {
4082	gen_helper_gvec_bfmla_idx, gen_helper_gvec_fmla_idx_h,
4083	gen_helper_gvec_fmla_idx_s, gen_helper_gvec_fmla_idx_d
4084	};
4085	TRANS_FEAT(FMLA_zzxz, aa64_sme_or_sve, do_fmla_zzxz, a, fmla_idx_fns[a->esz])
4086
4087	static gen_helper_gvec_4_ptr * const fmls_idx_fns[4][2] = {
4088	{ gen_helper_gvec_bfmls_idx, gen_helper_gvec_ah_bfmls_idx },
4089	{ gen_helper_gvec_fmls_idx_h, gen_helper_gvec_ah_fmls_idx_h },
4090	{ gen_helper_gvec_fmls_idx_s, gen_helper_gvec_ah_fmls_idx_s },
4091	{ gen_helper_gvec_fmls_idx_d, gen_helper_gvec_ah_fmls_idx_d },
4092	};
4093	TRANS_FEAT(FMLS_zzxz, aa64_sme_or_sve, do_fmla_zzxz, a,
4094	fmls_idx_fns[a->esz][s->fpcr_ah])
4095
4096	/*
4097	*** SVE Floating Point Multiply Indexed Group
4098	*/
4099
4100	static gen_helper_gvec_3_ptr * const fmul_idx_fns[4] = {
4101	gen_helper_gvec_fmul_idx_b16, gen_helper_gvec_fmul_idx_h,
4102	gen_helper_gvec_fmul_idx_s, gen_helper_gvec_fmul_idx_d,
4103	};
4104	TRANS_FEAT(FMUL_zzx, aa64_sme_or_sve, gen_gvec_fpst_zzz,
4105	fmul_idx_fns[a->esz], a->rd, a->rn, a->rm, a->index,
4106	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4107
4108	/*
4109	*** SVE Floating Point Fast Reduction Group
4110	*/
4111
4112	typedef void gen_helper_fp_reduce(TCGv_i64, TCGv_ptr, TCGv_ptr,
4113	TCGv_ptr, TCGv_i32);
4114
4115	static bool do_reduce(DisasContext s, arg_rpr_esz a,
4116	gen_helper_fp_reduce *fn)
4117	{
4118	unsigned vsz, p2vsz;
4119	TCGv_i32 t_desc;
4120	TCGv_ptr t_zn, t_pg, status;
4121	TCGv_i64 temp;
4122
4123	if (fn == NULL) {
4124	return false;
4125	}
4126	if (!sve_access_check(s)) {
4127	return true;
4128	}
4129
4130	vsz = vec_full_reg_size(s);
4131	p2vsz = pow2ceil(vsz);
4132	t_desc = tcg_constant_i32(simd_desc(vsz, vsz, p2vsz));
4133	temp = tcg_temp_new_i64();
4134	t_zn = tcg_temp_new_ptr();
4135	t_pg = tcg_temp_new_ptr();
4136
4137	tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn));
4138	tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg));
4139	status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4140
4141	fn(temp, t_zn, t_pg, status, t_desc);
4142
4143	write_fp_dreg(s, a->rd, temp);
4144	return true;
4145	}
4146
4147	#define DO_VPZ(NAME, name) \
4148	static gen_helper_fp_reduce * const name##_fns[4] = { \
4149	NULL, gen_helper_sve_##name##_h, \
4150	gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
4151	}; \
4152	TRANS_FEAT(NAME, aa64_sme_or_sve, do_reduce, a, name##_fns[a->esz])
4153
4154	#define DO_VPZ_AH(NAME, name) \
4155	static gen_helper_fp_reduce * const name##_fns[4] = { \
4156	NULL, gen_helper_sve_##name##_h, \
4157	gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
4158	}; \
4159	static gen_helper_fp_reduce * const name##_ah_fns[4] = { \
4160	NULL, gen_helper_sve_ah_##name##_h, \
4161	gen_helper_sve_ah_##name##_s, gen_helper_sve_ah_##name##_d, \
4162	}; \
4163	TRANS_FEAT(NAME, aa64_sme_or_sve, do_reduce, a, \
4164	s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz])
4165
4166	DO_VPZ(FADDV, faddv)
4167	DO_VPZ(FMINNMV, fminnmv)
4168	DO_VPZ(FMAXNMV, fmaxnmv)
4169	DO_VPZ_AH(FMINV, fminv)
4170	DO_VPZ_AH(FMAXV, fmaxv)
4171
4172	#undef DO_VPZ
4173
4174	static gen_helper_gvec_3_ptr * const faddqv_fns[4] = {
4175	NULL, gen_helper_sve2p1_faddqv_h,
4176	gen_helper_sve2p1_faddqv_s, gen_helper_sve2p1_faddqv_d,
4177	};
4178	TRANS_FEAT(FADDQV, aa64_sme2p1_or_sve2p1, gen_gvec_fpst_arg_zpz,
4179	faddqv_fns[a->esz], a, 0,
4180	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4181
4182	static gen_helper_gvec_3_ptr * const fmaxnmqv_fns[4] = {
4183	NULL, gen_helper_sve2p1_fmaxnmqv_h,
4184	gen_helper_sve2p1_fmaxnmqv_s, gen_helper_sve2p1_fmaxnmqv_d,
4185	};
4186	TRANS_FEAT(FMAXNMQV, aa64_sme2p1_or_sve2p1, gen_gvec_fpst_arg_zpz,
4187	fmaxnmqv_fns[a->esz], a, 0,
4188	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4189
4190	static gen_helper_gvec_3_ptr * const fminnmqv_fns[4] = {
4191	NULL, gen_helper_sve2p1_fminnmqv_h,
4192	gen_helper_sve2p1_fminnmqv_s, gen_helper_sve2p1_fminnmqv_d,
4193	};
4194	TRANS_FEAT(FMINNMQV, aa64_sme2p1_or_sve2p1, gen_gvec_fpst_arg_zpz,
4195	fminnmqv_fns[a->esz], a, 0,
4196	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4197
4198	static gen_helper_gvec_3_ptr * const fmaxqv_fns[4] = {
4199	NULL, gen_helper_sve2p1_fmaxqv_h,
4200	gen_helper_sve2p1_fmaxqv_s, gen_helper_sve2p1_fmaxqv_d,
4201	};
4202	static gen_helper_gvec_3_ptr * const fmaxqv_ah_fns[4] = {
4203	NULL, gen_helper_sve2p1_ah_fmaxqv_h,
4204	gen_helper_sve2p1_ah_fmaxqv_s, gen_helper_sve2p1_ah_fmaxqv_d,
4205	};
4206	TRANS_FEAT(FMAXQV, aa64_sme2p1_or_sve2p1, gen_gvec_fpst_arg_zpz,
4207	(s->fpcr_ah ? fmaxqv_ah_fns : fmaxqv_fns)[a->esz], a, 0,
4208	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4209
4210	static gen_helper_gvec_3_ptr * const fminqv_fns[4] = {
4211	NULL, gen_helper_sve2p1_fminqv_h,
4212	gen_helper_sve2p1_fminqv_s, gen_helper_sve2p1_fminqv_d,
4213	};
4214	static gen_helper_gvec_3_ptr * const fminqv_ah_fns[4] = {
4215	NULL, gen_helper_sve2p1_ah_fminqv_h,
4216	gen_helper_sve2p1_ah_fminqv_s, gen_helper_sve2p1_ah_fminqv_d,
4217	};
4218	TRANS_FEAT(FMINQV, aa64_sme2p1_or_sve2p1, gen_gvec_fpst_arg_zpz,
4219	(s->fpcr_ah ? fminqv_ah_fns : fminqv_fns)[a->esz], a, 0,
4220	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4221
4222	/*
4223	*** SVE Floating Point Unary Operations - Unpredicated Group
4224	*/
4225
4226	static gen_helper_gvec_2_ptr * const frecpe_fns[] = {
4227	NULL, gen_helper_gvec_frecpe_h,
4228	gen_helper_gvec_frecpe_s, gen_helper_gvec_frecpe_d,
4229	};
4230	static gen_helper_gvec_2_ptr * const frecpe_rpres_fns[] = {
4231	NULL, gen_helper_gvec_frecpe_h,
4232	gen_helper_gvec_frecpe_rpres_s, gen_helper_gvec_frecpe_d,
4233	};
4234	TRANS_FEAT(FRECPE, aa64_sme_or_sve, gen_gvec_fpst_ah_arg_zz,
4235	s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
4236	frecpe_rpres_fns[a->esz] : frecpe_fns[a->esz], a, 0)
4237
4238	static gen_helper_gvec_2_ptr * const frsqrte_fns[] = {
4239	NULL, gen_helper_gvec_frsqrte_h,
4240	gen_helper_gvec_frsqrte_s, gen_helper_gvec_frsqrte_d,
4241	};
4242	static gen_helper_gvec_2_ptr * const frsqrte_rpres_fns[] = {
4243	NULL, gen_helper_gvec_frsqrte_h,
4244	gen_helper_gvec_frsqrte_rpres_s, gen_helper_gvec_frsqrte_d,
4245	};
4246	TRANS_FEAT(FRSQRTE, aa64_sme_or_sve, gen_gvec_fpst_ah_arg_zz,
4247	s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
4248	frsqrte_rpres_fns[a->esz] : frsqrte_fns[a->esz], a, 0)
4249
4250	static bool do_f8cvt(DisasContext s, arg_rr_esz a,
4251	gen_helper_gvec_2_ptr *fn, bool issrc2, bool isodd)
4252	{
4253	if (fpmr_access_check(s) && sve_access_check(s)) {
4254	unsigned vsz = vec_full_reg_size(s);
4255	tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd),
4256	vec_full_reg_offset(s, a->rn),
4257	tcg_env, vsz, vsz,
4258	issrc2 \| (isodd << 1) \| (FPST_A64 << 2), fn);
4259	}
4260	return true;
4261	}
4262
4263	TRANS_FEAT_STREAMING_IF(F1CVT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4264	do_f8cvt, a, gen_helper_sve2_fcvt_hb, false, false)
4265	TRANS_FEAT_STREAMING_IF(F2CVT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4266	do_f8cvt, a, gen_helper_sve2_fcvt_hb, true, false)
4267	TRANS_FEAT_STREAMING_IF(F1CVTLT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4268	do_f8cvt, a, gen_helper_sve2_fcvt_hb, false, true)
4269	TRANS_FEAT_STREAMING_IF(F2CVTLT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4270	do_f8cvt, a, gen_helper_sve2_fcvt_hb, true, true)
4271
4272	TRANS_FEAT_STREAMING_IF(BF1CVT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4273	do_f8cvt, a, gen_helper_sve2_bfcvt, false, false)
4274	TRANS_FEAT_STREAMING_IF(BF2CVT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4275	do_f8cvt, a, gen_helper_sve2_bfcvt, true, false)
4276	TRANS_FEAT_STREAMING_IF(BF1CVTLT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4277	do_f8cvt, a, gen_helper_sve2_bfcvt, false, true)
4278	TRANS_FEAT_STREAMING_IF(BF2CVTLT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4279	do_f8cvt, a, gen_helper_sve2_bfcvt, true, true)
4280
4281	TRANS_FEAT_STREAMING_IF(FCVTN, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4282	do_f8cvt, a, gen_helper_sve2_fcvtn_bh, false, false)
4283	TRANS_FEAT_STREAMING_IF(BFCVTN, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4284	do_f8cvt, a, gen_helper_sve2_bfcvtn_bh, false, false)
4285	TRANS_FEAT_STREAMING_IF(FCVTNB, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4286	do_f8cvt, a, gen_helper_sve2_fcvtnb_bs, false, false)
4287	TRANS_FEAT_STREAMING_IF(FCVTNT, aa64_sme2_or_sve2_f8cvt, aa64_sme2,
4288	do_f8cvt, a, gen_helper_sve2_fcvtnt_bs, false, false)
4289
4290	/*
4291	*** SVE Floating Point Compare with Zero Group
4292	*/
4293
4294	static bool do_ppz_fp(DisasContext s, arg_rpr_esz a,
4295	gen_helper_gvec_3_ptr *fn)
4296	{
4297	if (fn == NULL) {
4298	return false;
4299	}
4300	if (sve_access_check(s)) {
4301	unsigned vsz = vec_full_reg_size(s);
4302	TCGv_ptr status =
4303	fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4304
4305	tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd),
4306	vec_full_reg_offset(s, a->rn),
4307	pred_full_reg_offset(s, a->pg),
4308	status, vsz, vsz, 0, fn);
4309	}
4310	return true;
4311	}
4312
4313	#define DO_PPZ(NAME, name) \
4314	static gen_helper_gvec_3_ptr * const name##_fns[] = { \
4315	NULL, gen_helper_sve_##name##_h, \
4316	gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
4317	}; \
4318	TRANS_FEAT(NAME, aa64_sme_or_sve, do_ppz_fp, a, name##_fns[a->esz])
4319
4320	DO_PPZ(FCMGE_ppz0, fcmge0)
4321	DO_PPZ(FCMGT_ppz0, fcmgt0)
4322	DO_PPZ(FCMLE_ppz0, fcmle0)
4323	DO_PPZ(FCMLT_ppz0, fcmlt0)
4324	DO_PPZ(FCMEQ_ppz0, fcmeq0)
4325	DO_PPZ(FCMNE_ppz0, fcmne0)
4326
4327	#undef DO_PPZ
4328
4329	/*
4330	*** SVE floating-point trig multiply-add coefficient
4331	*/
4332
4333	static gen_helper_gvec_3_ptr * const ftmad_fns[4] = {
4334	NULL, gen_helper_sve_ftmad_h,
4335	gen_helper_sve_ftmad_s, gen_helper_sve_ftmad_d,
4336	};
4337	TRANS_FEAT_NONSTREAMING(FTMAD, aa64_sve, gen_gvec_fpst_zzz,
4338	ftmad_fns[a->esz], a->rd, a->rn, a->rm,
4339	a->imm \| (s->fpcr_ah << 3),
4340	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4341
4342	/*
4343	*** SVE Floating Point Accumulating Reduction Group
4344	*/
4345
4346	static bool trans_FADDA(DisasContext s, arg_rprr_esz a)
4347	{
4348	typedef void fadda_fn(TCGv_i64, TCGv_i64, TCGv_ptr,
4349	TCGv_ptr, TCGv_ptr, TCGv_i32);
4350	static fadda_fn * const fns[3] = {
4351	gen_helper_sve_fadda_h,
4352	gen_helper_sve_fadda_s,
4353	gen_helper_sve_fadda_d,
4354	};
4355	unsigned vsz = vec_full_reg_size(s);
4356	TCGv_ptr t_rm, t_pg, t_fpst;
4357	TCGv_i64 t_val;
4358	TCGv_i32 t_desc;
4359
4360	if (a->esz == 0 \|\| !dc_isar_feature(aa64_sve, s)) {
4361	return false;
4362	}
4363	s->is_nonstreaming = true;
4364	if (!sve_access_check(s)) {
4365	return true;
4366	}
4367
4368	t_val = load_esz(tcg_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz);
4369	t_rm = tcg_temp_new_ptr();
4370	t_pg = tcg_temp_new_ptr();
4371	tcg_gen_addi_ptr(t_rm, tcg_env, vec_full_reg_offset(s, a->rm));
4372	tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg));
4373	t_fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4374	t_desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
4375
4376	fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc);
4377
4378	write_fp_dreg(s, a->rd, t_val);
4379	return true;
4380	}
4381
4382	/*
4383	*** SVE Floating Point Arithmetic - Unpredicated Group
4384	*/
4385
4386	#define DO_FP3(NAME, name) \
4387	static gen_helper_gvec_3_ptr * const name##_fns[4] = { \
4388	gen_helper_gvec_##name##_b16, gen_helper_gvec_##name##_h, \
4389	gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4390	}; \
4391	TRANS_FEAT(NAME, aa64_sme_or_sve, gen_gvec_fpst_arg_zzz, name##_fns[a->esz], a, 0)
4392
4393	#define DO_FP3_AH(NAME, name) \
4394	static gen_helper_gvec_3_ptr * const name##_fns[4] = { \
4395	NULL, gen_helper_gvec_##name##_h, \
4396	gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4397	}; \
4398	static gen_helper_gvec_3_ptr * const name##_ah_fns[4] = { \
4399	NULL, gen_helper_gvec_ah_##name##_h, \
4400	gen_helper_gvec_ah_##name##_s, gen_helper_gvec_ah_##name##_d \
4401	}; \
4402	TRANS_FEAT(NAME, aa64_sme_or_sve, gen_gvec_fpst_ah_arg_zzz, \
4403	s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz], a, 0)
4404
4405	DO_FP3(FADD_zzz, fadd)
4406	DO_FP3(FSUB_zzz, fsub)
4407	DO_FP3(FMUL_zzz, fmul)
4408	DO_FP3_AH(FRECPS, recps)
4409	DO_FP3_AH(FRSQRTS, rsqrts)
4410
4411	#undef DO_FP3
4412
4413	static gen_helper_gvec_3_ptr * const ftsmul_fns[4] = {
4414	NULL, gen_helper_gvec_ftsmul_h,
4415	gen_helper_gvec_ftsmul_s, gen_helper_gvec_ftsmul_d
4416	};
4417	TRANS_FEAT_NONSTREAMING(FTSMUL, aa64_sve, gen_gvec_fpst_arg_zzz,
4418	ftsmul_fns[a->esz], a, 0)
4419
4420	/*
4421	*** SVE Floating Point Arithmetic - Predicated Group
4422	*/
4423
4424	static gen_helper_gvec_4_ptr * const sve_fadd_zpzz_fns[4] = {
4425	NULL,
4426	gen_helper_sve_fadd_h,
4427	gen_helper_sve_fadd_s,
4428	gen_helper_sve_fadd_d
4429	};
4430	TRANS_FEAT(BFADD_zpzz, aa64_sve_b16b16, gen_gvec_fpst_arg_zpzz,
4431	gen_helper_sve_fadd_b16, a)
4432	TRANS_FEAT(FADD_zpzz, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4433	sve_fadd_zpzz_fns[a->esz], a)
4434
4435	static gen_helper_gvec_4_ptr * const sve_fsub_zpzz_fns[4] = {
4436	NULL,
4437	gen_helper_sve_fsub_h,
4438	gen_helper_sve_fsub_s,
4439	gen_helper_sve_fsub_d
4440	};
4441	TRANS_FEAT(BFSUB_zpzz, aa64_sve_b16b16, gen_gvec_fpst_arg_zpzz,
4442	gen_helper_sve_fsub_b16, a)
4443	TRANS_FEAT(FSUB_zpzz, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4444	sve_fsub_zpzz_fns[a->esz], a)
4445
4446	static gen_helper_gvec_4_ptr * const sve_fmul_zpzz_fns[4] = {
4447	NULL,
4448	gen_helper_sve_fmul_h,
4449	gen_helper_sve_fmul_s,
4450	gen_helper_sve_fmul_d
4451	};
4452	TRANS_FEAT(BFMUL_zpzz, aa64_sve_b16b16, gen_gvec_fpst_arg_zpzz,
4453	gen_helper_sve_fmul_b16, a)
4454	TRANS_FEAT(FMUL_zpzz, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4455	sve_fmul_zpzz_fns[a->esz], a)
4456
4457	static gen_helper_gvec_4_ptr * const sve_fmin_fns[4][2] = {
4458	{ NULL, NULL },
4459	{ gen_helper_sve_fmin_h, gen_helper_sve_ah_fmin_h },
4460	{ gen_helper_sve_fmin_s, gen_helper_sve_ah_fmin_s },
4461	{ gen_helper_sve_fmin_d, gen_helper_sve_ah_fmin_d },
4462	};
4463	TRANS_FEAT(BFMIN_zpzz, aa64_sve_b16b16, gen_gvec_fpst_arg_zpzz,
4464	s->fpcr_ah ? gen_helper_sve_ah_fmin_b16 : gen_helper_sve_fmin_b16, a)
4465	TRANS_FEAT(FMIN_zpzz, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4466	sve_fmin_fns[a->esz][s->fpcr_ah], a)
4467
4468	static gen_helper_gvec_4_ptr * const sve_fmax_fns[4][2] = {
4469	{ NULL, NULL },
4470	{ gen_helper_sve_fmax_h, gen_helper_sve_ah_fmax_h },
4471	{ gen_helper_sve_fmax_s, gen_helper_sve_ah_fmax_s },
4472	{ gen_helper_sve_fmax_d, gen_helper_sve_ah_fmax_d },
4473	};
4474	TRANS_FEAT(BFMAX_zpzz, aa64_sve_b16b16, gen_gvec_fpst_arg_zpzz,
4475	s->fpcr_ah ? gen_helper_sve_ah_fmax_b16 : gen_helper_sve_fmax_b16, a)
4476	TRANS_FEAT(FMAX_zpzz, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4477	sve_fmax_fns[a->esz][s->fpcr_ah], a)
4478
4479	static gen_helper_gvec_4_ptr * const sve_fmaxnum_fns[4] = {
4480	NULL,
4481	gen_helper_sve_fmaxnum_h,
4482	gen_helper_sve_fmaxnum_s,
4483	gen_helper_sve_fmaxnum_d
4484	};
4485	TRANS_FEAT(BFMAXNM_zpzz, aa64_sve_b16b16, gen_gvec_fpst_arg_zpzz,
4486	gen_helper_sve_fmaxnum_b16, a)
4487	TRANS_FEAT(FMAXNM_zpzz, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4488	sve_fmaxnum_fns[a->esz], a)
4489
4490	static gen_helper_gvec_4_ptr * const sve_fminnum_fns[4] = {
4491	NULL,
4492	gen_helper_sve_fminnum_h,
4493	gen_helper_sve_fminnum_s,
4494	gen_helper_sve_fminnum_d
4495	};
4496	TRANS_FEAT(BFMINNM_zpzz, aa64_sve_b16b16, gen_gvec_fpst_arg_zpzz,
4497	gen_helper_sve_fminnum_b16, a)
4498	TRANS_FEAT(FMINNM_zpzz, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4499	sve_fminnum_fns[a->esz], a)
4500
4501	static gen_helper_gvec_4_ptr * const sve_fabd_zpzz_fns[4][2] = {
4502	{ NULL, NULL },
4503	{ gen_helper_sve_fabd_h, gen_helper_sve_ah_fabd_h },
4504	{ gen_helper_sve_fabd_s, gen_helper_sve_ah_fabd_s },
4505	{ gen_helper_sve_fabd_d, gen_helper_sve_ah_fabd_d },
4506	};
4507	TRANS_FEAT(FABD, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4508	sve_fabd_zpzz_fns[a->esz][s->fpcr_ah], a)
4509
4510	static gen_helper_gvec_4_ptr * const sve_fscalbn_zpzz_fns[4] = {
4511	NULL,
4512	gen_helper_sve_fscalbn_h,
4513	gen_helper_sve_fscalbn_s,
4514	gen_helper_sve_fscalbn_d,
4515	};
4516	TRANS_FEAT(FSCALE, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4517	sve_fscalbn_zpzz_fns[a->esz], a)
4518	TRANS_FEAT(BFSCALE, aa64_sve_bfscale, gen_gvec_fpst_arg_zpzz,
4519	gen_helper_sve_fscalbn_b16, a)
4520
4521	static gen_helper_gvec_4_ptr * const sve_fdiv_zpzz_fns[4] = {
4522	NULL,
4523	gen_helper_sve_fdiv_h,
4524	gen_helper_sve_fdiv_s,
4525	gen_helper_sve_fdiv_d,
4526	};
4527	TRANS_FEAT(FDIV, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4528	sve_fdiv_zpzz_fns[a->esz], a)
4529
4530	static gen_helper_gvec_4_ptr * const sve_fmulx_zpzz_fns[4] = {
4531	NULL,
4532	gen_helper_sve_fmulx_h,
4533	gen_helper_sve_fmulx_s,
4534	gen_helper_sve_fmulx_d,
4535	};
4536	TRANS_FEAT(FMULX, aa64_sme_or_sve, gen_gvec_fpst_arg_zpzz,
4537	sve_fmulx_zpzz_fns[a->esz], a)
4538
4539	static gen_helper_gvec_4_ptr * const sve2_famax_zpzz_fns[4] = {
4540	NULL,
4541	gen_helper_sve2_famax_h,
4542	gen_helper_sve2_famax_s,
4543	gen_helper_sve2_famax_d
4544	};
4545	TRANS_FEAT_STREAMING_IF(FAMAX, aa64_sme2_or_sve2_faminmax, aa64_sme2,
4546	gen_gvec_fpst_arg_zpzz,
4547	sve2_famax_zpzz_fns[a->esz], a)
4548
4549	static gen_helper_gvec_4_ptr * const sve2_famin_zpzz_fns[4] = {
4550	NULL,
4551	gen_helper_sve2_famin_h,
4552	gen_helper_sve2_famin_s,
4553	gen_helper_sve2_famin_d
4554	};
4555	TRANS_FEAT_STREAMING_IF(FAMIN, aa64_sme2_or_sve2_faminmax, aa64_sme2,
4556	gen_gvec_fpst_arg_zpzz,
4557	sve2_famin_zpzz_fns[a->esz], a)
4558
4559	typedef void gen_helper_sve_fp2scalar(TCGv_ptr, TCGv_ptr, TCGv_ptr,
4560	TCGv_i64, TCGv_ptr, TCGv_i32);
4561
4562	static void do_fp_scalar(DisasContext *s, int zd, int zn, int pg, bool is_fp16,
4563	TCGv_i64 scalar, gen_helper_sve_fp2scalar *fn)
4564	{
4565	unsigned vsz = vec_full_reg_size(s);
4566	TCGv_ptr t_zd, t_zn, t_pg, status;
4567	TCGv_i32 desc;
4568
4569	t_zd = tcg_temp_new_ptr();
4570	t_zn = tcg_temp_new_ptr();
4571	t_pg = tcg_temp_new_ptr();
4572	tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, zd));
4573	tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, zn));
4574	tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
4575
4576	status = fpstatus_ptr(is_fp16 ? FPST_A64_F16 : FPST_A64);
4577	desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
4578	fn(t_zd, t_zn, t_pg, scalar, status, desc);
4579	}
4580
4581	static bool do_fp_imm(DisasContext s, arg_rpri_esz a, uint64_t imm,
4582	gen_helper_sve_fp2scalar *fn)
4583	{
4584	if (fn == NULL) {
4585	return false;
4586	}
4587	if (sve_access_check(s)) {
4588	do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16,
4589	tcg_constant_i64(imm), fn);
4590	}
4591	return true;
4592	}
4593
4594	#define DO_FP_IMM(NAME, name, const0, const1) \
4595	static gen_helper_sve_fp2scalar * const name##_fns[4] = { \
4596	NULL, gen_helper_sve_##name##_h, \
4597	gen_helper_sve_##name##_s, \
4598	gen_helper_sve_##name##_d \
4599	}; \
4600	static uint64_t const name##_const[4][2] = { \
4601	{ -1, -1 }, \
4602	{ float16_##const0, float16_##const1 }, \
4603	{ float32_##const0, float32_##const1 }, \
4604	{ float64_##const0, float64_##const1 }, \
4605	}; \
4606	TRANS_FEAT(NAME##_zpzi, aa64_sme_or_sve, do_fp_imm, a, \
4607	name##_const[a->esz][a->imm], name##_fns[a->esz])
4608
4609	#define DO_FP_AH_IMM(NAME, name, const0, const1) \
4610	static gen_helper_sve_fp2scalar * const name##_fns[4] = { \
4611	NULL, gen_helper_sve_##name##_h, \
4612	gen_helper_sve_##name##_s, \
4613	gen_helper_sve_##name##_d \
4614	}; \
4615	static gen_helper_sve_fp2scalar * const name##_ah_fns[4] = { \
4616	NULL, gen_helper_sve_ah_##name##_h, \
4617	gen_helper_sve_ah_##name##_s, \
4618	gen_helper_sve_ah_##name##_d \
4619	}; \
4620	static uint64_t const name##_const[4][2] = { \
4621	{ -1, -1 }, \
4622	{ float16_##const0, float16_##const1 }, \
4623	{ float32_##const0, float32_##const1 }, \
4624	{ float64_##const0, float64_##const1 }, \
4625	}; \
4626	TRANS_FEAT(NAME##_zpzi, aa64_sme_or_sve, do_fp_imm, a, \
4627	name##_const[a->esz][a->imm], \
4628	s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz])
4629
4630	DO_FP_IMM(FADD, fadds, half, one)
4631	DO_FP_IMM(FSUB, fsubs, half, one)
4632	DO_FP_IMM(FMUL, fmuls, half, two)
4633	DO_FP_IMM(FSUBR, fsubrs, half, one)
4634	DO_FP_IMM(FMAXNM, fmaxnms, zero, one)
4635	DO_FP_IMM(FMINNM, fminnms, zero, one)
4636	DO_FP_AH_IMM(FMAX, fmaxs, zero, one)
4637	DO_FP_AH_IMM(FMIN, fmins, zero, one)
4638
4639	#undef DO_FP_IMM
4640
4641	static bool do_fp_cmp(DisasContext s, arg_rprr_esz a,
4642	gen_helper_gvec_4_ptr *fn)
4643	{
4644	if (fn == NULL) {
4645	return false;
4646	}
4647	if (sve_access_check(s)) {
4648	unsigned vsz = vec_full_reg_size(s);
4649	TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4650	tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd),
4651	vec_full_reg_offset(s, a->rn),
4652	vec_full_reg_offset(s, a->rm),
4653	pred_full_reg_offset(s, a->pg),
4654	status, vsz, vsz, 0, fn);
4655	}
4656	return true;
4657	}
4658
4659	#define DO_FPCMP(NAME, name) \
4660	static gen_helper_gvec_4_ptr * const name##_fns[4] = { \
4661	NULL, gen_helper_sve_##name##_h, \
4662	gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4663	}; \
4664	TRANS_FEAT(NAME##_ppzz, aa64_sme_or_sve, do_fp_cmp, a, name##_fns[a->esz])
4665
4666	DO_FPCMP(FCMGE, fcmge)
4667	DO_FPCMP(FCMGT, fcmgt)
4668	DO_FPCMP(FCMEQ, fcmeq)
4669	DO_FPCMP(FCMNE, fcmne)
4670	DO_FPCMP(FCMUO, fcmuo)
4671	DO_FPCMP(FACGE, facge)
4672	DO_FPCMP(FACGT, facgt)
4673
4674	#undef DO_FPCMP
4675
4676	static gen_helper_gvec_4_ptr * const fcadd_fns[] = {
4677	NULL, gen_helper_sve_fcadd_h,
4678	gen_helper_sve_fcadd_s, gen_helper_sve_fcadd_d,
4679	};
4680	TRANS_FEAT(FCADD, aa64_sme_or_sve, gen_gvec_fpst_zzzp, fcadd_fns[a->esz],
4681	a->rd, a->rn, a->rm, a->pg, a->rot \| (s->fpcr_ah << 1),
4682	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4683
4684	static bool do_fmla_zpzzz(DisasContext s, arg_rprrr_esz a,
4685	gen_helper_gvec_5_ptr *fn)
4686	{
4687	/* These insns use MO_8 to encode BFloat16 */
4688	if (a->esz == MO_8 && !dc_isar_feature(aa64_sve_b16b16, s)) {
4689	return false;
4690	}
4691	return gen_gvec_fpst_zzzzp(s, fn, a->rd, a->rn, a->rm, a->ra, a->pg, 0,
4692	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4693	}
4694
4695	#define DO_FMLA(NAME, name, ah_name) \
4696	static gen_helper_gvec_5_ptr * const name##_fns[4] = { \
4697	gen_helper_sve_##name##_b16, gen_helper_sve_##name##_h, \
4698	gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4699	}; \
4700	static gen_helper_gvec_5_ptr * const name##_ah_fns[4] = { \
4701	gen_helper_sve_##ah_name##_b16, gen_helper_sve_##ah_name##_h, \
4702	gen_helper_sve_##ah_name##_s, gen_helper_sve_##ah_name##_d \
4703	}; \
4704	TRANS_FEAT(NAME, aa64_sme_or_sve, do_fmla_zpzzz, a, \
4705	s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz])
4706
4707	/* We don't need an ah_fmla_zpzzz because fmla doesn't negate anything */
4708	DO_FMLA(FMLA_zpzzz, fmla_zpzzz, fmla_zpzzz)
4709	DO_FMLA(FMLS_zpzzz, fmls_zpzzz, ah_fmls_zpzzz)
4710	DO_FMLA(FNMLA_zpzzz, fnmla_zpzzz, ah_fnmla_zpzzz)
4711	DO_FMLA(FNMLS_zpzzz, fnmls_zpzzz, ah_fnmls_zpzzz)
4712
4713	#undef DO_FMLA
4714
4715	static gen_helper_gvec_5_ptr * const fcmla_fns[4] = {
4716	NULL, gen_helper_sve_fcmla_zpzzz_h,
4717	gen_helper_sve_fcmla_zpzzz_s, gen_helper_sve_fcmla_zpzzz_d,
4718	};
4719	TRANS_FEAT(FCMLA_zpzzz, aa64_sme_or_sve, gen_gvec_fpst_zzzzp, fcmla_fns[a->esz],
4720	a->rd, a->rn, a->rm, a->ra, a->pg, a->rot \| (s->fpcr_ah << 2),
4721	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4722
4723	static gen_helper_gvec_4_ptr * const fcmla_idx_fns[4] = {
4724	NULL, gen_helper_gvec_fcmlah_idx, gen_helper_gvec_fcmlas_idx, NULL
4725	};
4726	TRANS_FEAT(FCMLA_zzxz, aa64_sme_or_sve, gen_gvec_fpst_zzzz, fcmla_idx_fns[a->esz],
4727	a->rd, a->rn, a->rm, a->ra, a->index * 4 + a->rot,
4728	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4729
4730	/*
4731	*** SVE Floating Point Unary Operations Predicated Group
4732	*/
4733
4734	TRANS_FEAT(FCVT_sh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4735	gen_helper_sve_fcvt_sh, a, 0, FPST_A64)
4736	TRANS_FEAT(FCVT_sh_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4737	gen_helper_sve_fcvt_sh, a, 1, FPST_A64)
4738
4739	TRANS_FEAT(FCVT_hs_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4740	gen_helper_sve_fcvt_hs, a, 0, FPST_A64_F16)
4741	TRANS_FEAT(FCVT_hs_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4742	gen_helper_sve_fcvt_hs, a, 1, FPST_A64_F16)
4743
4744	TRANS_FEAT(BFCVT_m, aa64_sme_sve_bf16, gen_gvec_fpst_arg_zpz,
4745	gen_helper_sve_bfcvt, a, 0,
4746	s->fpcr_ah ? FPST_AH : FPST_A64)
4747	TRANS_FEAT(BFCVT_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4748	gen_helper_sve_bfcvt, a, 1,
4749	s->fpcr_ah ? FPST_AH : FPST_A64)
4750
4751	TRANS_FEAT(FCVT_dh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4752	gen_helper_sve_fcvt_dh, a, 0, FPST_A64)
4753	TRANS_FEAT(FCVT_dh_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4754	gen_helper_sve_fcvt_dh, a, 1, FPST_A64)
4755
4756	TRANS_FEAT(FCVT_hd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4757	gen_helper_sve_fcvt_hd, a, 0, FPST_A64_F16)
4758	TRANS_FEAT(FCVT_hd_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4759	gen_helper_sve_fcvt_hd, a, 1, FPST_A64_F16)
4760
4761	TRANS_FEAT(FCVT_ds_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4762	gen_helper_sve_fcvt_ds, a, 0, FPST_A64)
4763	TRANS_FEAT(FCVT_ds_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4764	gen_helper_sve_fcvt_ds, a, 1, FPST_A64)
4765
4766	TRANS_FEAT(FCVT_sd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4767	gen_helper_sve_fcvt_sd, a, 0, FPST_A64)
4768	TRANS_FEAT(FCVT_sd_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4769	gen_helper_sve_fcvt_sd, a, 1, FPST_A64)
4770
4771	TRANS_FEAT(FCVTZS_hh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4772	gen_helper_sve_fcvtzs_hh, a, 0, FPST_A64_F16)
4773	TRANS_FEAT(FCVTZU_hh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4774	gen_helper_sve_fcvtzu_hh, a, 0, FPST_A64_F16)
4775	TRANS_FEAT(FCVTZS_hs_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4776	gen_helper_sve_fcvtzs_hs, a, 0, FPST_A64_F16)
4777	TRANS_FEAT(FCVTZU_hs_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4778	gen_helper_sve_fcvtzu_hs, a, 0, FPST_A64_F16)
4779	TRANS_FEAT(FCVTZS_hd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4780	gen_helper_sve_fcvtzs_hd, a, 0, FPST_A64_F16)
4781	TRANS_FEAT(FCVTZU_hd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4782	gen_helper_sve_fcvtzu_hd, a, 0, FPST_A64_F16)
4783
4784	TRANS_FEAT(FCVTZS_ss_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4785	gen_helper_sve_fcvtzs_ss, a, 0, FPST_A64)
4786	TRANS_FEAT(FCVTZU_ss_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4787	gen_helper_sve_fcvtzu_ss, a, 0, FPST_A64)
4788	TRANS_FEAT(FCVTZS_sd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4789	gen_helper_sve_fcvtzs_sd, a, 0, FPST_A64)
4790	TRANS_FEAT(FCVTZU_sd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4791	gen_helper_sve_fcvtzu_sd, a, 0, FPST_A64)
4792	TRANS_FEAT(FCVTZS_ds_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4793	gen_helper_sve_fcvtzs_ds, a, 0, FPST_A64)
4794	TRANS_FEAT(FCVTZU_ds_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4795	gen_helper_sve_fcvtzu_ds, a, 0, FPST_A64)
4796
4797	TRANS_FEAT(FCVTZS_dd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4798	gen_helper_sve_fcvtzs_dd, a, 0, FPST_A64)
4799	TRANS_FEAT(FCVTZU_dd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4800	gen_helper_sve_fcvtzu_dd, a, 0, FPST_A64)
4801
4802	TRANS_FEAT(FCVTZS_hh_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4803	gen_helper_sve_fcvtzs_hh, a, 1, FPST_A64_F16)
4804	TRANS_FEAT(FCVTZU_hh_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4805	gen_helper_sve_fcvtzu_hh, a, 1, FPST_A64_F16)
4806	TRANS_FEAT(FCVTZS_hs_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4807	gen_helper_sve_fcvtzs_hs, a, 1, FPST_A64_F16)
4808	TRANS_FEAT(FCVTZU_hs_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4809	gen_helper_sve_fcvtzu_hs, a, 1, FPST_A64_F16)
4810	TRANS_FEAT(FCVTZS_hd_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4811	gen_helper_sve_fcvtzs_hd, a, 1, FPST_A64_F16)
4812	TRANS_FEAT(FCVTZU_hd_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4813	gen_helper_sve_fcvtzu_hd, a, 1, FPST_A64_F16)
4814
4815	TRANS_FEAT(FCVTZS_ss_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4816	gen_helper_sve_fcvtzs_ss, a, 1, FPST_A64)
4817	TRANS_FEAT(FCVTZU_ss_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4818	gen_helper_sve_fcvtzu_ss, a, 1, FPST_A64)
4819	TRANS_FEAT(FCVTZS_sd_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4820	gen_helper_sve_fcvtzs_sd, a, 1, FPST_A64)
4821	TRANS_FEAT(FCVTZU_sd_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4822	gen_helper_sve_fcvtzu_sd, a, 1, FPST_A64)
4823	TRANS_FEAT(FCVTZS_ds_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4824	gen_helper_sve_fcvtzs_ds, a, 1, FPST_A64)
4825	TRANS_FEAT(FCVTZU_ds_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4826	gen_helper_sve_fcvtzu_ds, a, 1, FPST_A64)
4827
4828	TRANS_FEAT(FCVTZS_dd_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4829	gen_helper_sve_fcvtzs_dd, a, 1, FPST_A64)
4830	TRANS_FEAT(FCVTZU_dd_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4831	gen_helper_sve_fcvtzu_dd, a, 1, FPST_A64)
4832
4833	static gen_helper_gvec_3_ptr * const frint_fns[] = {
4834	NULL,
4835	gen_helper_sve_frint_h,
4836	gen_helper_sve_frint_s,
4837	gen_helper_sve_frint_d
4838	};
4839	TRANS_FEAT(FRINTI_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4840	frint_fns[a->esz], a, 0,
4841	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4842	TRANS_FEAT(FRINTI_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4843	frint_fns[a->esz], a, 1,
4844	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4845
4846	static gen_helper_gvec_3_ptr * const frintx_fns[] = {
4847	NULL,
4848	gen_helper_sve_frintx_h,
4849	gen_helper_sve_frintx_s,
4850	gen_helper_sve_frintx_d
4851	};
4852	TRANS_FEAT(FRINTX_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4853	frintx_fns[a->esz], a, 0,
4854	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4855	TRANS_FEAT(FRINTX_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4856	frintx_fns[a->esz], a, 1,
4857	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4858
4859	static bool do_frint_mode(DisasContext s, arg_rpr_esz a,
4860	ARMFPRounding mode, int data,
4861	gen_helper_gvec_3_ptr *fn)
4862	{
4863	unsigned vsz;
4864	TCGv_i32 tmode;
4865	TCGv_ptr status;
4866
4867	if (fn == NULL) {
4868	return false;
4869	}
4870	if (!sve_access_check(s)) {
4871	return true;
4872	}
4873
4874	vsz = vec_full_reg_size(s);
4875	status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4876	tmode = gen_set_rmode(mode, status);
4877
4878	tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
4879	vec_full_reg_offset(s, a->rn),
4880	pred_full_reg_offset(s, a->pg),
4881	status, vsz, vsz, data, fn);
4882
4883	gen_restore_rmode(tmode, status);
4884	return true;
4885	}
4886
4887	TRANS_FEAT(FRINTN_m, aa64_sme_or_sve, do_frint_mode, a,
4888	FPROUNDING_TIEEVEN, 0, frint_fns[a->esz])
4889	TRANS_FEAT(FRINTP_m, aa64_sme_or_sve, do_frint_mode, a,
4890	FPROUNDING_POSINF, 0, frint_fns[a->esz])
4891	TRANS_FEAT(FRINTM_m, aa64_sme_or_sve, do_frint_mode, a,
4892	FPROUNDING_NEGINF, 0, frint_fns[a->esz])
4893	TRANS_FEAT(FRINTZ_m, aa64_sme_or_sve, do_frint_mode, a,
4894	FPROUNDING_ZERO, 0, frint_fns[a->esz])
4895	TRANS_FEAT(FRINTA_m, aa64_sme_or_sve, do_frint_mode, a,
4896	FPROUNDING_TIEAWAY, 0, frint_fns[a->esz])
4897
4898	TRANS_FEAT(FRINTN_z, aa64_sme2p2_or_sve2p2, do_frint_mode, a,
4899	FPROUNDING_TIEEVEN, 1, frint_fns[a->esz])
4900	TRANS_FEAT(FRINTP_z, aa64_sme2p2_or_sve2p2, do_frint_mode, a,
4901	FPROUNDING_POSINF, 1, frint_fns[a->esz])
4902	TRANS_FEAT(FRINTM_z, aa64_sme2p2_or_sve2p2, do_frint_mode, a,
4903	FPROUNDING_NEGINF, 1, frint_fns[a->esz])
4904	TRANS_FEAT(FRINTZ_z, aa64_sme2p2_or_sve2p2, do_frint_mode, a,
4905	FPROUNDING_ZERO, 1, frint_fns[a->esz])
4906	TRANS_FEAT(FRINTA_z, aa64_sme2p2_or_sve2p2, do_frint_mode, a,
4907	FPROUNDING_TIEAWAY, 1, frint_fns[a->esz])
4908
4909	TRANS_FEAT(FRINT32X_s_m, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4910	gen_helper_sve2p2_frint32_s, a, 0, FPST_A64)
4911	TRANS_FEAT(FRINT32X_d_m, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4912	gen_helper_sve2p2_frint32_d, a, 0, FPST_A64)
4913	TRANS_FEAT(FRINT64X_s_m, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4914	gen_helper_sve2p2_frint64_s, a, 0, FPST_A64)
4915	TRANS_FEAT(FRINT64X_d_m, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4916	gen_helper_sve2p2_frint64_d, a, 0, FPST_A64)
4917
4918	TRANS_FEAT(FRINT32X_s_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4919	gen_helper_sve2p2_frint32_s, a, 1, FPST_A64)
4920	TRANS_FEAT(FRINT32X_d_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4921	gen_helper_sve2p2_frint32_d, a, 1, FPST_A64)
4922	TRANS_FEAT(FRINT64X_s_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4923	gen_helper_sve2p2_frint64_s, a, 1, FPST_A64)
4924	TRANS_FEAT(FRINT64X_d_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4925	gen_helper_sve2p2_frint64_d, a, 1, FPST_A64)
4926
4927	TRANS_FEAT(FRINT32Z_s_m, aa64_sme2p2_or_sve2p2, do_frint_mode,
4928	a, FPROUNDING_ZERO, 0, gen_helper_sve2p2_frint32_s)
4929	TRANS_FEAT(FRINT32Z_d_m, aa64_sme2p2_or_sve2p2, do_frint_mode,
4930	a, FPROUNDING_ZERO, 0, gen_helper_sve2p2_frint32_d)
4931	TRANS_FEAT(FRINT64Z_s_m, aa64_sme2p2_or_sve2p2, do_frint_mode,
4932	a, FPROUNDING_ZERO, 0, gen_helper_sve2p2_frint64_s)
4933	TRANS_FEAT(FRINT64Z_d_m, aa64_sme2p2_or_sve2p2, do_frint_mode,
4934	a, FPROUNDING_ZERO, 0, gen_helper_sve2p2_frint64_d)
4935
4936	TRANS_FEAT(FRINT32Z_s_z, aa64_sme2p2_or_sve2p2, do_frint_mode,
4937	a, FPROUNDING_ZERO, 1, gen_helper_sve2p2_frint32_s)
4938	TRANS_FEAT(FRINT32Z_d_z, aa64_sme2p2_or_sve2p2, do_frint_mode,
4939	a, FPROUNDING_ZERO, 1, gen_helper_sve2p2_frint32_d)
4940	TRANS_FEAT(FRINT64Z_s_z, aa64_sme2p2_or_sve2p2, do_frint_mode,
4941	a, FPROUNDING_ZERO, 1, gen_helper_sve2p2_frint64_s)
4942	TRANS_FEAT(FRINT64Z_d_z, aa64_sme2p2_or_sve2p2, do_frint_mode,
4943	a, FPROUNDING_ZERO, 1, gen_helper_sve2p2_frint64_d)
4944
4945	static gen_helper_gvec_3_ptr * const frecpx_fns[] = {
4946	NULL, gen_helper_sve_frecpx_h,
4947	gen_helper_sve_frecpx_s, gen_helper_sve_frecpx_d,
4948	};
4949	TRANS_FEAT(FRECPX_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4950	frecpx_fns[a->esz], a, 0, select_ah_fpst(s, a->esz))
4951	TRANS_FEAT(FRECPX_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4952	frecpx_fns[a->esz], a, 1, select_ah_fpst(s, a->esz))
4953
4954	static gen_helper_gvec_3_ptr * const fsqrt_fns[] = {
4955	NULL, gen_helper_sve_fsqrt_h,
4956	gen_helper_sve_fsqrt_s, gen_helper_sve_fsqrt_d,
4957	};
4958	TRANS_FEAT(FSQRT_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4959	fsqrt_fns[a->esz], a, 0,
4960	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4961	TRANS_FEAT(FSQRT_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
4962	fsqrt_fns[a->esz], a, 1,
4963	a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4964
4965	TRANS_FEAT(SCVTF_hh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4966	gen_helper_sve_scvt_hh, a, 0, FPST_A64_F16)
4967	TRANS_FEAT(SCVTF_sh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4968	gen_helper_sve_scvt_sh, a, 0, FPST_A64_F16)
4969	TRANS_FEAT(SCVTF_dh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4970	gen_helper_sve_scvt_dh, a, 0, FPST_A64_F16)
4971
4972	TRANS_FEAT(SCVTF_ss_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4973	gen_helper_sve_scvt_ss, a, 0, FPST_A64)
4974	TRANS_FEAT(SCVTF_ds_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4975	gen_helper_sve_scvt_ds, a, 0, FPST_A64)
4976
4977	TRANS_FEAT(SCVTF_sd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4978	gen_helper_sve_scvt_sd, a, 0, FPST_A64)
4979	TRANS_FEAT(SCVTF_dd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4980	gen_helper_sve_scvt_dd, a, 0, FPST_A64)
4981
4982	TRANS_FEAT(UCVTF_hh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4983	gen_helper_sve_ucvt_hh, a, 0, FPST_A64_F16)
4984	TRANS_FEAT(UCVTF_sh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4985	gen_helper_sve_ucvt_sh, a, 0, FPST_A64_F16)
4986	TRANS_FEAT(UCVTF_dh_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4987	gen_helper_sve_ucvt_dh, a, 0, FPST_A64_F16)
4988
4989	TRANS_FEAT(UCVTF_ss_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4990	gen_helper_sve_ucvt_ss, a, 0, FPST_A64)
4991	TRANS_FEAT(UCVTF_ds_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4992	gen_helper_sve_ucvt_ds, a, 0, FPST_A64)
4993	TRANS_FEAT(UCVTF_sd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4994	gen_helper_sve_ucvt_sd, a, 0, FPST_A64)
4995
4996	TRANS_FEAT(UCVTF_dd_m, aa64_sme_or_sve, gen_gvec_fpst_arg_zpz,
4997	gen_helper_sve_ucvt_dd, a, 0, FPST_A64)
4998
4999	TRANS_FEAT(SCVTF_hh_z, aa64_sme2p2_or_sve2p2, gen_gvec_fpst_arg_zpz,
5000	gen_helper_sve_scvt_hh, a, 1, FPST_A64_F16)

Showing first 5,000 of 8,976 lines. View raw