translate-neon.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 3,374 lines 101 KB

Raw

1	/*
2	* ARM translation: AArch32 Neon instructions
3	*
4	* Copyright (c) 2003 Fabrice Bellard
5	* Copyright (c) 2005-2007 CodeSourcery
6	* Copyright (c) 2007 OpenedHand, Ltd.
7	* Copyright (c) 2020 Linaro, Ltd.
8	*
9	* This library is free software; you can redistribute it and/or
10	* modify it under the terms of the GNU Lesser General Public
11	* License as published by the Free Software Foundation; either
12	* version 2.1 of the License, or (at your option) any later version.
13	*
14	* This library is distributed in the hope that it will be useful,
15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17	* Lesser General Public License for more details.
18	*
19	* You should have received a copy of the GNU Lesser General Public
20	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
21	*/
22
23	#include "qemu/osdep.h"
24	#include "translate.h"
25	#include "translate-a32.h"
26
27	/* Include the generated Neon decoder */
28	#include "decode-neon-dp.c.inc"
29	#include "decode-neon-ls.c.inc"
30	#include "decode-neon-shared.c.inc"
31
32	static TCGv_ptr vfp_reg_ptr(bool dp, int reg)
33	{
34	TCGv_ptr ret = tcg_temp_new_ptr();
35	tcg_gen_addi_ptr(ret, tcg_env, vfp_reg_offset(dp, reg));
36	return ret;
37	}
38
39	static void neon_load_element(TCGv_i32 var, int reg, int ele, MemOp mop)
40	{
41	long offset = neon_element_offset(reg, ele, mop & MO_SIZE);
42
43	switch (mop) {
44	case MO_UB:
45	tcg_gen_ld8u_i32(var, tcg_env, offset);
46	break;
47	case MO_UW:
48	tcg_gen_ld16u_i32(var, tcg_env, offset);
49	break;
50	case MO_UL:
51	tcg_gen_ld_i32(var, tcg_env, offset);
52	break;
53	default:
54	g_assert_not_reached();
55	}
56	}
57
58	static void neon_load_element64(TCGv_i64 var, int reg, int ele, MemOp mop)
59	{
60	long offset = neon_element_offset(reg, ele, mop & MO_SIZE);
61
62	switch (mop) {
63	case MO_UB:
64	tcg_gen_ld8u_i64(var, tcg_env, offset);
65	break;
66	case MO_UW:
67	tcg_gen_ld16u_i64(var, tcg_env, offset);
68	break;
69	case MO_UL:
70	tcg_gen_ld32u_i64(var, tcg_env, offset);
71	break;
72	case MO_UQ:
73	tcg_gen_ld_i64(var, tcg_env, offset);
74	break;
75	default:
76	g_assert_not_reached();
77	}
78	}
79
80	static void neon_store_element(int reg, int ele, MemOp size, TCGv_i32 var)
81	{
82	long offset = neon_element_offset(reg, ele, size);
83
84	switch (size) {
85	case MO_8:
86	tcg_gen_st8_i32(var, tcg_env, offset);
87	break;
88	case MO_16:
89	tcg_gen_st16_i32(var, tcg_env, offset);
90	break;
91	case MO_32:
92	tcg_gen_st_i32(var, tcg_env, offset);
93	break;
94	default:
95	g_assert_not_reached();
96	}
97	}
98
99	static void neon_store_element64(int reg, int ele, MemOp size, TCGv_i64 var)
100	{
101	long offset = neon_element_offset(reg, ele, size);
102
103	switch (size) {
104	case MO_8:
105	tcg_gen_st8_i64(var, tcg_env, offset);
106	break;
107	case MO_16:
108	tcg_gen_st16_i64(var, tcg_env, offset);
109	break;
110	case MO_32:
111	tcg_gen_st32_i64(var, tcg_env, offset);
112	break;
113	case MO_64:
114	tcg_gen_st_i64(var, tcg_env, offset);
115	break;
116	default:
117	g_assert_not_reached();
118	}
119	}
120
121	/*
122	* Return true if a Neon insn is OK to access the registers indicated
123	* by regmask, false if it should UNDEF. This checks whether the
124	* D16-D31 regs are implemented by the CPU.
125	* Note that Neon insns accessing D16..D31 do not need to check D32DIS.
126	*
127	* @regmask should be the logical OR of the Dregs being accessed.
128	*/
129	static bool neon_dregs_ok(DisasContext *s, int dregmask)
130	{
131	return !(dregmask & s->invalid_neon_dreg_mask);
132	}
133
134	static bool do_neon_ddda(DisasContext *s, int q, int vd, int vn, int vm,
135	int data, gen_helper_gvec_4 *fn_gvec)
136	{
137	/* UNDEF accesses to D16-D31 if they don't exist. */
138	if (!neon_dregs_ok(s, vd \| vn \| vm)) {
139	return false;
140	}
141
142	/*
143	* UNDEF accesses to odd registers for each bit of Q.
144	* Q will be 0b111 for all Q-reg instructions, otherwise
145	* when we have mixed Q- and D-reg inputs.
146	*/
147	if (((vd & 1) * 4 \| (vn & 1) * 2 \| (vm & 1)) & q) {
148	return false;
149	}
150
151	if (!neon_access_check(s)) {
152	return true;
153	}
154
155	int opr_sz = q ? 16 : 8;
156	tcg_gen_gvec_4_ool(vfp_reg_offset(1, vd),
157	vfp_reg_offset(1, vn),
158	vfp_reg_offset(1, vm),
159	vfp_reg_offset(1, vd),
160	opr_sz, opr_sz, data, fn_gvec);
161	return true;
162	}
163
164	static bool do_neon_ddda_env(DisasContext *s, int q, int vd, int vn, int vm,
165	int data, gen_helper_gvec_4_ptr *fn_gvec)
166	{
167	/* UNDEF accesses to D16-D31 if they don't exist. */
168	if (!neon_dregs_ok(s, vd \| vn \| vm)) {
169	return false;
170	}
171
172	/*
173	* UNDEF accesses to odd registers for each bit of Q.
174	* Q will be 0b111 for all Q-reg instructions, otherwise
175	* when we have mixed Q- and D-reg inputs.
176	*/
177	if (((vd & 1) * 4 \| (vn & 1) * 2 \| (vm & 1)) & q) {
178	return false;
179	}
180
181	if (!neon_access_check(s)) {
182	return true;
183	}
184
185	int opr_sz = q ? 16 : 8;
186	tcg_gen_gvec_4_ptr(vfp_reg_offset(1, vd),
187	vfp_reg_offset(1, vn),
188	vfp_reg_offset(1, vm),
189	vfp_reg_offset(1, vd),
190	tcg_env,
191	opr_sz, opr_sz, data, fn_gvec);
192	return true;
193	}
194
195	static bool do_neon_ddda_fpst(DisasContext *s, int q, int vd, int vn, int vm,
196	int data, ARMFPStatusFlavour fp_flavour,
197	gen_helper_gvec_4_ptr *fn_gvec_ptr)
198	{
199	/* UNDEF accesses to D16-D31 if they don't exist. */
200	if (!neon_dregs_ok(s, vd \| vn \| vm)) {
201	return false;
202	}
203
204	/*
205	* UNDEF accesses to odd registers for each bit of Q.
206	* Q will be 0b111 for all Q-reg instructions, otherwise
207	* when we have mixed Q- and D-reg inputs.
208	*/
209	if (((vd & 1) * 4 \| (vn & 1) * 2 \| (vm & 1)) & q) {
210	return false;
211	}
212
213	if (!neon_access_check(s)) {
214	return true;
215	}
216
217	int opr_sz = q ? 16 : 8;
218	TCGv_ptr fpst = fpstatus_ptr(fp_flavour);
219
220	tcg_gen_gvec_4_ptr(vfp_reg_offset(1, vd),
221	vfp_reg_offset(1, vn),
222	vfp_reg_offset(1, vm),
223	vfp_reg_offset(1, vd),
224	fpst, opr_sz, opr_sz, data, fn_gvec_ptr);
225	return true;
226	}
227
228	static bool trans_VCMLA(DisasContext s, arg_VCMLA a)
229	{
230	if (!dc_isar_feature(aa32_vcma, s)) {
231	return false;
232	}
233	if (a->size == MO_16) {
234	if (!dc_isar_feature(aa32_fp16_arith, s)) {
235	return false;
236	}
237	return do_neon_ddda_fpst(s, a->q * 7, a->vd, a->vn, a->vm, a->rot,
238	FPST_STD_F16, gen_helper_gvec_fcmlah);
239	}
240	return do_neon_ddda_fpst(s, a->q * 7, a->vd, a->vn, a->vm, a->rot,
241	FPST_STD, gen_helper_gvec_fcmlas);
242	}
243
244	static bool trans_VCADD(DisasContext s, arg_VCADD a)
245	{
246	int opr_sz;
247	TCGv_ptr fpst;
248	gen_helper_gvec_3_ptr *fn_gvec_ptr;
249
250	if (!dc_isar_feature(aa32_vcma, s)
251	\|\| (a->size == MO_16 && !dc_isar_feature(aa32_fp16_arith, s))) {
252	return false;
253	}
254
255	/* UNDEF accesses to D16-D31 if they don't exist. */
256	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
257	return false;
258	}
259
260	if ((a->vn \| a->vm \| a->vd) & a->q) {
261	return false;
262	}
263
264	if (!neon_access_check(s)) {
265	return true;
266	}
267
268	opr_sz = (1 + a->q) * 8;
269	fpst = fpstatus_ptr(a->size == MO_16 ? FPST_STD_F16 : FPST_STD);
270	fn_gvec_ptr = (a->size == MO_16) ?
271	gen_helper_gvec_fcaddh : gen_helper_gvec_fcadds;
272	tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd),
273	vfp_reg_offset(1, a->vn),
274	vfp_reg_offset(1, a->vm),
275	fpst, opr_sz, opr_sz, a->rot,
276	fn_gvec_ptr);
277	return true;
278	}
279
280	static bool trans_VSDOT(DisasContext s, arg_VSDOT a)
281	{
282	if (!dc_isar_feature(aa32_dp, s)) {
283	return false;
284	}
285	return do_neon_ddda(s, a->q * 7, a->vd, a->vn, a->vm, 0,
286	gen_helper_gvec_sdot_4b);
287	}
288
289	static bool trans_VUDOT(DisasContext s, arg_VUDOT a)
290	{
291	if (!dc_isar_feature(aa32_dp, s)) {
292	return false;
293	}
294	return do_neon_ddda(s, a->q * 7, a->vd, a->vn, a->vm, 0,
295	gen_helper_gvec_udot_4b);
296	}
297
298	static bool trans_VUSDOT(DisasContext s, arg_VUSDOT a)
299	{
300	if (!dc_isar_feature(aa32_i8mm, s)) {
301	return false;
302	}
303	return do_neon_ddda(s, a->q * 7, a->vd, a->vn, a->vm, 0,
304	gen_helper_gvec_usdot_4b);
305	}
306
307	static bool trans_VDOT_b16(DisasContext s, arg_VDOT_b16 a)
308	{
309	if (!dc_isar_feature(aa32_bf16, s)) {
310	return false;
311	}
312	return do_neon_ddda_env(s, a->q * 7, a->vd, a->vn, a->vm, 0,
313	gen_helper_gvec_bfdot);
314	}
315
316	static bool trans_VFML(DisasContext s, arg_VFML a)
317	{
318	int opr_sz;
319
320	if (!dc_isar_feature(aa32_fhm, s)) {
321	return false;
322	}
323
324	/* UNDEF accesses to D16-D31 if they don't exist. */
325	if (!neon_dregs_ok(s, a->vd)) {
326	return false;
327	}
328
329	if (a->vd & a->q) {
330	return false;
331	}
332
333	if (!neon_access_check(s)) {
334	return true;
335	}
336
337	opr_sz = (1 + a->q) * 8;
338	tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd),
339	vfp_reg_offset(a->q, a->vn),
340	vfp_reg_offset(a->q, a->vm),
341	tcg_env, opr_sz, opr_sz, a->s, /* is_2 == 0 */
342	gen_helper_gvec_fmlal_a32);
343	return true;
344	}
345
346	static bool trans_VCMLA_scalar(DisasContext s, arg_VCMLA_scalar a)
347	{
348	int data = (a->index << 2) \| a->rot;
349
350	if (!dc_isar_feature(aa32_vcma, s)) {
351	return false;
352	}
353	if (a->size == MO_16) {
354	if (!dc_isar_feature(aa32_fp16_arith, s)) {
355	return false;
356	}
357	return do_neon_ddda_fpst(s, a->q * 6, a->vd, a->vn, a->vm, data,
358	FPST_STD_F16, gen_helper_gvec_fcmlah_idx);
359	}
360	return do_neon_ddda_fpst(s, a->q * 6, a->vd, a->vn, a->vm, data,
361	FPST_STD, gen_helper_gvec_fcmlas_idx);
362	}
363
364	static bool trans_VSDOT_scalar(DisasContext s, arg_VSDOT_scalar a)
365	{
366	if (!dc_isar_feature(aa32_dp, s)) {
367	return false;
368	}
369	return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
370	gen_helper_gvec_sdot_idx_4b);
371	}
372
373	static bool trans_VUDOT_scalar(DisasContext s, arg_VUDOT_scalar a)
374	{
375	if (!dc_isar_feature(aa32_dp, s)) {
376	return false;
377	}
378	return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
379	gen_helper_gvec_udot_idx_4b);
380	}
381
382	static bool trans_VUSDOT_scalar(DisasContext s, arg_VUSDOT_scalar a)
383	{
384	if (!dc_isar_feature(aa32_i8mm, s)) {
385	return false;
386	}
387	return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
388	gen_helper_gvec_usdot_idx_4b);
389	}
390
391	static bool trans_VSUDOT_scalar(DisasContext s, arg_VSUDOT_scalar a)
392	{
393	if (!dc_isar_feature(aa32_i8mm, s)) {
394	return false;
395	}
396	return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
397	gen_helper_gvec_sudot_idx_4b);
398	}
399
400	static bool trans_VDOT_b16_scal(DisasContext s, arg_VDOT_b16_scal a)
401	{
402	if (!dc_isar_feature(aa32_bf16, s)) {
403	return false;
404	}
405	return do_neon_ddda_env(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
406	gen_helper_gvec_bfdot_idx);
407	}
408
409	static bool trans_VFML_scalar(DisasContext s, arg_VFML_scalar a)
410	{
411	int opr_sz;
412	int dregmask = a->vd \| (a->q ? a->vn : 0);
413
414	if (!dc_isar_feature(aa32_fhm, s)) {
415	return false;
416	}
417
418	/* UNDEF accesses to D16-D31 if they don't exist. */
419	if (!neon_dregs_ok(s, dregmask)) {
420	return false;
421	}
422
423	if (a->vd & a->q) {
424	return false;
425	}
426
427	if (!neon_access_check(s)) {
428	return true;
429	}
430
431	opr_sz = (1 + a->q) * 8;
432	tcg_gen_gvec_3_ptr(vfp_reg_offset(1, a->vd),
433	vfp_reg_offset(a->q, a->vn),
434	vfp_reg_offset(a->q, a->rm),
435	tcg_env, opr_sz, opr_sz,
436	(a->index << 2) \| a->s, /* is_2 == 0 */
437	gen_helper_gvec_fmlal_idx_a32);
438	return true;
439	}
440
441	static struct {
442	int nregs;
443	int interleave;
444	int spacing;
445	} const neon_ls_element_type[11] = {
446	{1, 4, 1},
447	{1, 4, 2},
448	{4, 1, 1},
449	{2, 2, 2},
450	{1, 3, 1},
451	{1, 3, 2},
452	{3, 1, 1},
453	{1, 1, 1},
454	{1, 2, 1},
455	{1, 2, 2},
456	{2, 1, 1}
457	};
458
459	static void gen_neon_ldst_base_update(DisasContext *s, int rm, int rn,
460	int stride)
461	{
462	if (rm != 15) {
463	TCGv_i32 base;
464
465	base = load_reg(s, rn);
466	if (rm == 13) {
467	tcg_gen_addi_i32(base, base, stride);
468	} else {
469	TCGv_i32 index;
470	index = load_reg(s, rm);
471	tcg_gen_add_i32(base, base, index);
472	}
473	store_reg(s, rn, base);
474	}
475	}
476
477	static bool trans_VLDST_multiple(DisasContext s, arg_VLDST_multiple a)
478	{
479	/* Neon load/store multiple structures */
480	int nregs, interleave, spacing, reg, n;
481	MemOp mop, align, endian;
482	int mmu_idx = get_mem_index(s);
483	int size = a->size;
484	TCGv_i64 tmp64;
485	TCGv_i32 addr;
486
487	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
488	return false;
489	}
490
491	/* UNDEF accesses to D16-D31 if they don't exist */
492	if (!neon_dregs_ok(s, a->vd)) {
493	return false;
494	}
495	if (a->itype > 10) {
496	return false;
497	}
498	/* Catch UNDEF cases for bad values of align field */
499	switch (a->itype & 0xc) {
500	case 4:
501	if (a->align >= 2) {
502	return false;
503	}
504	break;
505	case 8:
506	if (a->align == 3) {
507	return false;
508	}
509	break;
510	default:
511	break;
512	}
513	nregs = neon_ls_element_type[a->itype].nregs;
514	interleave = neon_ls_element_type[a->itype].interleave;
515	spacing = neon_ls_element_type[a->itype].spacing;
516	if (size == 3 && (interleave \| spacing) != 1) {
517	return false;
518	}
519
520	if (!neon_access_check(s)) {
521	return true;
522	}
523
524	/* For our purposes, bytes are always little-endian. */
525	endian = s->be_data;
526	if (size == 0) {
527	endian = MO_LE;
528	}
529
530	/* Enforce alignment requested by the instruction */
531	if (a->align) {
532	align = pow2_align(a->align + 2); /* 4 ** a->align */
533	} else {
534	align = MO_ALIGN \| (s->align_mem ? 0 : MO_ALIGN_TLB_ONLY);
535	}
536
537	/*
538	* Consecutive little-endian elements from a single register
539	* can be promoted to a larger little-endian operation.
540	*/
541	if (interleave == 1 && endian == MO_LE) {
542	/* Retain any natural alignment. */
543	if (align == MO_ALIGN) {
544	align = pow2_align(size);
545	}
546	size = 3;
547	}
548
549	tmp64 = tcg_temp_new_i64();
550	addr = tcg_temp_new_i32();
551	load_reg_var(s, addr, a->rn);
552
553	mop = endian \| size \| align;
554	for (reg = 0; reg < nregs; reg++) {
555	for (n = 0; n < 8 >> size; n++) {
556	int xs;
557	for (xs = 0; xs < interleave; xs++) {
558	int tt = a->vd + reg + spacing * xs;
559
560	if (a->l) {
561	gen_aa32_ld_internal_i64(s, tmp64, addr, mmu_idx, mop);
562	neon_store_element64(tt, n, size, tmp64);
563	} else {
564	neon_load_element64(tmp64, tt, n, size);
565	gen_aa32_st_internal_i64(s, tmp64, addr, mmu_idx, mop);
566	}
567	tcg_gen_addi_i32(addr, addr, 1 << size);
568
569	/* Subsequent memory operations inherit alignment */
570	mop &= ~MO_AMASK;
571	}
572	}
573	}
574
575	gen_neon_ldst_base_update(s, a->rm, a->rn, nregs * interleave * 8);
576	return true;
577	}
578
579	static bool trans_VLD_all_lanes(DisasContext s, arg_VLD_all_lanes a)
580	{
581	/* Neon load single structure to all lanes */
582	int reg, stride, vec_size;
583	int vd = a->vd;
584	int size = a->size;
585	int nregs = a->n + 1;
586	TCGv_i32 addr, tmp;
587	MemOp mop, align;
588
589	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
590	return false;
591	}
592
593	/* UNDEF accesses to D16-D31 if they don't exist */
594	if (!neon_dregs_ok(s, a->vd)) {
595	return false;
596	}
597
598	align = 0;
599	if (size == 3) {
600	if (nregs != 4 \|\| a->a == 0) {
601	return false;
602	}
603	/* For VLD4 size == 3 a == 1 means 32 bits at 16 byte alignment */
604	size = MO_32;
605	align = MO_ALIGN_16;
606	} else if (a->a) {
607	switch (nregs) {
608	case 1:
609	if (size == 0) {
610	return false;
611	}
612	align = MO_ALIGN;
613	break;
614	case 2:
615	align = pow2_align(size + 1);
616	break;
617	case 3:
618	return false;
619	case 4:
620	if (size == 2) {
621	align = pow2_align(3);
622	} else {
623	align = pow2_align(size + 2);
624	}
625	break;
626	default:
627	g_assert_not_reached();
628	}
629	}
630
631	if (!neon_access_check(s)) {
632	return true;
633	}
634
635	/*
636	* VLD1 to all lanes: T bit indicates how many Dregs to write.
637	* VLD2/3/4 to all lanes: T bit indicates register stride.
638	*/
639	stride = a->t ? 2 : 1;
640	vec_size = nregs == 1 ? stride * 8 : 8;
641	mop = size \| align;
642	tmp = tcg_temp_new_i32();
643	addr = tcg_temp_new_i32();
644	load_reg_var(s, addr, a->rn);
645	for (reg = 0; reg < nregs; reg++) {
646	gen_aa32_ld_i32(s, tmp, addr, get_mem_index(s), mop);
647	if ((vd & 1) && vec_size == 16) {
648	/*
649	* We cannot write 16 bytes at once because the
650	* destination is unaligned.
651	*/
652	tcg_gen_gvec_dup_i32(size, neon_full_reg_offset(vd),
653	8, 8, tmp);
654	tcg_gen_gvec_mov(0, neon_full_reg_offset(vd + 1),
655	neon_full_reg_offset(vd), 8, 8);
656	} else {
657	tcg_gen_gvec_dup_i32(size, neon_full_reg_offset(vd),
658	vec_size, vec_size, tmp);
659	}
660	tcg_gen_addi_i32(addr, addr, 1 << size);
661	vd += stride;
662
663	/* Subsequent memory operations inherit alignment */
664	mop &= ~MO_AMASK;
665	}
666
667	gen_neon_ldst_base_update(s, a->rm, a->rn, (1 << size) * nregs);
668
669	return true;
670	}
671
672	static bool trans_VLDST_single(DisasContext s, arg_VLDST_single a)
673	{
674	/* Neon load/store single structure to one lane */
675	int reg;
676	int nregs = a->n + 1;
677	int vd = a->vd;
678	TCGv_i32 addr, tmp;
679	MemOp mop;
680
681	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
682	return false;
683	}
684
685	/* UNDEF accesses to D16-D31 if they don't exist */
686	if (!neon_dregs_ok(s, a->vd)) {
687	return false;
688	}
689
690	/* Catch the UNDEF cases. This is unavoidably a bit messy. */
691	switch (nregs) {
692	case 1:
693	if (a->stride != 1) {
694	return false;
695	}
696	if (((a->align & (1 << a->size)) != 0) \|\|
697	(a->size == 2 && (a->align == 1 \|\| a->align == 2))) {
698	return false;
699	}
700	break;
701	case 2:
702	if (a->size == 2 && (a->align & 2) != 0) {
703	return false;
704	}
705	break;
706	case 3:
707	if (a->align != 0) {
708	return false;
709	}
710	break;
711	case 4:
712	if (a->size == 2 && a->align == 3) {
713	return false;
714	}
715	break;
716	default:
717	g_assert_not_reached();
718	}
719	if ((vd + a->stride * (nregs - 1)) > 31) {
720	/*
721	* Attempts to write off the end of the register file are
722	* UNPREDICTABLE; we choose to UNDEF because otherwise we would
723	* access off the end of the array that holds the register data.
724	*/
725	return false;
726	}
727
728	if (!neon_access_check(s)) {
729	return true;
730	}
731
732	/* Pick up SCTLR settings */
733	mop = finalize_memop(s, a->size);
734
735	if (a->align) {
736	MemOp align_op;
737
738	switch (nregs) {
739	case 1:
740	/* For VLD1, use natural alignment. */
741	align_op = MO_ALIGN;
742	break;
743	case 2:
744	/* For VLD2, use double alignment. */
745	align_op = pow2_align(a->size + 1);
746	break;
747	case 4:
748	if (a->size == MO_32) {
749	/*
750	* For VLD4.32, align = 1 is double alignment, align = 2 is
751	* quad alignment; align = 3 is rejected above.
752	*/
753	align_op = pow2_align(a->size + a->align);
754	} else {
755	/* For VLD4.8 and VLD.16, we want quad alignment. */
756	align_op = pow2_align(a->size + 2);
757	}
758	break;
759	default:
760	/* For VLD3, the alignment field is zero and rejected above. */
761	g_assert_not_reached();
762	}
763
764	mop = (mop & ~MO_AMASK) \| align_op;
765	}
766
767	tmp = tcg_temp_new_i32();
768	addr = tcg_temp_new_i32();
769	load_reg_var(s, addr, a->rn);
770
771	for (reg = 0; reg < nregs; reg++) {
772	if (a->l) {
773	gen_aa32_ld_internal_i32(s, tmp, addr, get_mem_index(s), mop);
774	neon_store_element(vd, a->reg_idx, a->size, tmp);
775	} else { /* Store */
776	neon_load_element(tmp, vd, a->reg_idx, a->size);
777	gen_aa32_st_internal_i32(s, tmp, addr, get_mem_index(s), mop);
778	}
779	vd += a->stride;
780	tcg_gen_addi_i32(addr, addr, 1 << a->size);
781
782	/* Subsequent memory operations inherit alignment */
783	mop &= ~MO_AMASK;
784	}
785
786	gen_neon_ldst_base_update(s, a->rm, a->rn, (1 << a->size) * nregs);
787
788	return true;
789	}
790
791	static bool do_3same(DisasContext s, arg_3same a, GVecGen3Fn fn)
792	{
793	int vec_size = a->q ? 16 : 8;
794	int rd_ofs = neon_full_reg_offset(a->vd);
795	int rn_ofs = neon_full_reg_offset(a->vn);
796	int rm_ofs = neon_full_reg_offset(a->vm);
797
798	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
799	return false;
800	}
801
802	/* UNDEF accesses to D16-D31 if they don't exist. */
803	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
804	return false;
805	}
806
807	if ((a->vn \| a->vm \| a->vd) & a->q) {
808	return false;
809	}
810
811	if (!neon_access_check(s)) {
812	return true;
813	}
814
815	fn(a->size, rd_ofs, rn_ofs, rm_ofs, vec_size, vec_size);
816	return true;
817	}
818
819	#define DO_3SAME(INSN, FUNC) \
820	static bool trans_##INSN##_3s(DisasContext s, arg_3same a) \
821	{ \
822	return do_3same(s, a, FUNC); \
823	}
824
825	DO_3SAME(VADD, tcg_gen_gvec_add)
826	DO_3SAME(VSUB, tcg_gen_gvec_sub)
827	DO_3SAME(VAND, tcg_gen_gvec_and)
828	DO_3SAME(VBIC, tcg_gen_gvec_andc)
829	DO_3SAME(VORR, tcg_gen_gvec_or)
830	DO_3SAME(VORN, tcg_gen_gvec_orc)
831	DO_3SAME(VEOR, tcg_gen_gvec_xor)
832	DO_3SAME(VSHL_S, gen_gvec_sshl)
833	DO_3SAME(VSHL_U, gen_gvec_ushl)
834	DO_3SAME(VQADD_S, gen_gvec_sqadd_qc)
835	DO_3SAME(VQADD_U, gen_gvec_uqadd_qc)
836	DO_3SAME(VQSUB_S, gen_gvec_sqsub_qc)
837	DO_3SAME(VQSUB_U, gen_gvec_uqsub_qc)
838	DO_3SAME(VRSHL_S, gen_gvec_srshl)
839	DO_3SAME(VRSHL_U, gen_gvec_urshl)
840	DO_3SAME(VQSHL_S, gen_neon_sqshl)
841	DO_3SAME(VQSHL_U, gen_neon_uqshl)
842	DO_3SAME(VQRSHL_S, gen_neon_sqrshl)
843	DO_3SAME(VQRSHL_U, gen_neon_uqrshl)
844
845	/* These insns are all gvec_bitsel but with the inputs in various orders. */
846	#define DO_3SAME_BITSEL(INSN, O1, O2, O3) \
847	static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \
848	uint32_t rn_ofs, uint32_t rm_ofs, \
849	uint32_t oprsz, uint32_t maxsz) \
850	{ \
851	tcg_gen_gvec_bitsel(vece, rd_ofs, O1, O2, O3, oprsz, maxsz); \
852	} \
853	DO_3SAME(INSN, gen_##INSN##_3s)
854
855	DO_3SAME_BITSEL(VBSL, rd_ofs, rn_ofs, rm_ofs)
856	DO_3SAME_BITSEL(VBIT, rm_ofs, rn_ofs, rd_ofs)
857	DO_3SAME_BITSEL(VBIF, rm_ofs, rd_ofs, rn_ofs)
858
859	#define DO_3SAME_NO_SZ_3(INSN, FUNC) \
860	static bool trans_##INSN##_3s(DisasContext s, arg_3same a) \
861	{ \
862	if (a->size == 3) { \
863	return false; \
864	} \
865	return do_3same(s, a, FUNC); \
866	}
867
868	DO_3SAME_NO_SZ_3(VMAX_S, tcg_gen_gvec_smax)
869	DO_3SAME_NO_SZ_3(VMAX_U, tcg_gen_gvec_umax)
870	DO_3SAME_NO_SZ_3(VMIN_S, tcg_gen_gvec_smin)
871	DO_3SAME_NO_SZ_3(VMIN_U, tcg_gen_gvec_umin)
872	DO_3SAME_NO_SZ_3(VMUL, tcg_gen_gvec_mul)
873	DO_3SAME_NO_SZ_3(VMLA, gen_gvec_mla)
874	DO_3SAME_NO_SZ_3(VMLS, gen_gvec_mls)
875	DO_3SAME_NO_SZ_3(VTST, gen_gvec_cmtst)
876	DO_3SAME_NO_SZ_3(VABD_S, gen_gvec_sabd)
877	DO_3SAME_NO_SZ_3(VABA_S, gen_gvec_saba)
878	DO_3SAME_NO_SZ_3(VABD_U, gen_gvec_uabd)
879	DO_3SAME_NO_SZ_3(VABA_U, gen_gvec_uaba)
880	DO_3SAME_NO_SZ_3(VPADD, gen_gvec_addp)
881	DO_3SAME_NO_SZ_3(VPMAX_S, gen_gvec_smaxp)
882	DO_3SAME_NO_SZ_3(VPMIN_S, gen_gvec_sminp)
883	DO_3SAME_NO_SZ_3(VPMAX_U, gen_gvec_umaxp)
884	DO_3SAME_NO_SZ_3(VPMIN_U, gen_gvec_uminp)
885	DO_3SAME_NO_SZ_3(VHADD_S, gen_gvec_shadd)
886	DO_3SAME_NO_SZ_3(VHADD_U, gen_gvec_uhadd)
887	DO_3SAME_NO_SZ_3(VHSUB_S, gen_gvec_shsub)
888	DO_3SAME_NO_SZ_3(VHSUB_U, gen_gvec_uhsub)
889	DO_3SAME_NO_SZ_3(VRHADD_S, gen_gvec_srhadd)
890	DO_3SAME_NO_SZ_3(VRHADD_U, gen_gvec_urhadd)
891
892	#define DO_3SAME_CMP(INSN, COND) \
893	static void gen_##INSN##_3s(unsigned vece, uint32_t rd_ofs, \
894	uint32_t rn_ofs, uint32_t rm_ofs, \
895	uint32_t oprsz, uint32_t maxsz) \
896	{ \
897	tcg_gen_gvec_cmp(COND, vece, rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz); \
898	} \
899	DO_3SAME_NO_SZ_3(INSN, gen_##INSN##_3s)
900
901	DO_3SAME_CMP(VCGT_S, TCG_COND_GT)
902	DO_3SAME_CMP(VCGT_U, TCG_COND_GTU)
903	DO_3SAME_CMP(VCGE_S, TCG_COND_GE)
904	DO_3SAME_CMP(VCGE_U, TCG_COND_GEU)
905	DO_3SAME_CMP(VCEQ, TCG_COND_EQ)
906
907	#define WRAP_OOL_FN(WRAPNAME, FUNC) \
908	static void WRAPNAME(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, \
909	uint32_t rm_ofs, uint32_t oprsz, uint32_t maxsz) \
910	{ \
911	tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, oprsz, maxsz, 0, FUNC); \
912	}
913
914	WRAP_OOL_FN(gen_VMUL_p_3s, gen_helper_gvec_pmul_b)
915
916	static bool trans_VMUL_p_3s(DisasContext s, arg_3same a)
917	{
918	if (a->size != 0) {
919	return false;
920	}
921	return do_3same(s, a, gen_VMUL_p_3s);
922	}
923
924	#define DO_VQRDMLAH(INSN, FUNC) \
925	static bool trans_##INSN##_3s(DisasContext s, arg_3same a) \
926	{ \
927	if (!dc_isar_feature(aa32_rdm, s)) { \
928	return false; \
929	} \
930	if (a->size != 1 && a->size != 2) { \
931	return false; \
932	} \
933	return do_3same(s, a, FUNC); \
934	}
935
936	DO_VQRDMLAH(VQRDMLAH, gen_gvec_sqrdmlah_qc)
937	DO_VQRDMLAH(VQRDMLSH, gen_gvec_sqrdmlsh_qc)
938
939	#define DO_SHA1(NAME, FUNC) \
940	WRAP_OOL_FN(gen_##NAME##_3s, FUNC) \
941	static bool trans_##NAME##_3s(DisasContext s, arg_3same a) \
942	{ \
943	if (!dc_isar_feature(aa32_sha1, s)) { \
944	return false; \
945	} \
946	return do_3same(s, a, gen_##NAME##_3s); \
947	}
948
949	DO_SHA1(SHA1C, gen_helper_crypto_sha1c)
950	DO_SHA1(SHA1P, gen_helper_crypto_sha1p)
951	DO_SHA1(SHA1M, gen_helper_crypto_sha1m)
952	DO_SHA1(SHA1SU0, gen_helper_crypto_sha1su0)
953
954	#define DO_SHA2(NAME, FUNC) \
955	WRAP_OOL_FN(gen_##NAME##_3s, FUNC) \
956	static bool trans_##NAME##_3s(DisasContext s, arg_3same a) \
957	{ \
958	if (!dc_isar_feature(aa32_sha2, s)) { \
959	return false; \
960	} \
961	return do_3same(s, a, gen_##NAME##_3s); \
962	}
963
964	DO_SHA2(SHA256H, gen_helper_crypto_sha256h)
965	DO_SHA2(SHA256H2, gen_helper_crypto_sha256h2)
966	DO_SHA2(SHA256SU1, gen_helper_crypto_sha256su1)
967
968	/*
969	* Some helper functions need to be passed the tcg_env. In order
970	* to use those with the gvec APIs like tcg_gen_gvec_3() we need
971	* to create wrapper functions whose prototype is a NeonGenTwoOpFn()
972	* and which call a NeonGenTwoOpEnvFn().
973	*/
974	#define WRAP_ENV_FN(WRAPNAME, FUNC) \
975	static void WRAPNAME(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m) \
976	{ \
977	FUNC(d, tcg_env, n, m); \
978	}
979
980	#define DO_3SAME_VQDMULH(INSN, FUNC) \
981	static bool trans_##INSN##_3s(DisasContext s, arg_3same a) \
982	{ return a->size >= 1 && a->size <= 2 && do_3same(s, a, FUNC); }
983
984	DO_3SAME_VQDMULH(VQDMULH, gen_gvec_sqdmulh_qc)
985	DO_3SAME_VQDMULH(VQRDMULH, gen_gvec_sqrdmulh_qc)
986
987	#define WRAP_FP_GVEC(WRAPNAME, FPST, FUNC) \
988	static void WRAPNAME(unsigned vece, uint32_t rd_ofs, \
989	uint32_t rn_ofs, uint32_t rm_ofs, \
990	uint32_t oprsz, uint32_t maxsz) \
991	{ \
992	TCGv_ptr fpst = fpstatus_ptr(FPST); \
993	tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, fpst, \
994	oprsz, maxsz, 0, FUNC); \
995	}
996
997	#define DO_3S_FP_GVEC(INSN,SFUNC,HFUNC) \
998	WRAP_FP_GVEC(gen_##INSN##_fp32_3s, FPST_STD, SFUNC) \
999	WRAP_FP_GVEC(gen_##INSN##_fp16_3s, FPST_STD_F16, HFUNC) \
1000	static bool trans_##INSN##_fp_3s(DisasContext s, arg_3same a) \
1001	{ \
1002	if (a->size == MO_16) { \
1003	if (!dc_isar_feature(aa32_fp16_arith, s)) { \
1004	return false; \
1005	} \
1006	return do_3same(s, a, gen_##INSN##_fp16_3s); \
1007	} \
1008	return do_3same(s, a, gen_##INSN##_fp32_3s); \
1009	}
1010
1011
1012	DO_3S_FP_GVEC(VADD, gen_helper_gvec_fadd_s, gen_helper_gvec_fadd_h)
1013	DO_3S_FP_GVEC(VSUB, gen_helper_gvec_fsub_s, gen_helper_gvec_fsub_h)
1014	DO_3S_FP_GVEC(VABD, gen_helper_gvec_fabd_s, gen_helper_gvec_fabd_h)
1015	DO_3S_FP_GVEC(VMUL, gen_helper_gvec_fmul_s, gen_helper_gvec_fmul_h)
1016	DO_3S_FP_GVEC(VCEQ, gen_helper_gvec_fceq_s, gen_helper_gvec_fceq_h)
1017	DO_3S_FP_GVEC(VCGE, gen_helper_gvec_fcge_s, gen_helper_gvec_fcge_h)
1018	DO_3S_FP_GVEC(VCGT, gen_helper_gvec_fcgt_s, gen_helper_gvec_fcgt_h)
1019	DO_3S_FP_GVEC(VACGE, gen_helper_gvec_facge_s, gen_helper_gvec_facge_h)
1020	DO_3S_FP_GVEC(VACGT, gen_helper_gvec_facgt_s, gen_helper_gvec_facgt_h)
1021	DO_3S_FP_GVEC(VMAX, gen_helper_gvec_fmax_s, gen_helper_gvec_fmax_h)
1022	DO_3S_FP_GVEC(VMIN, gen_helper_gvec_fmin_s, gen_helper_gvec_fmin_h)
1023	DO_3S_FP_GVEC(VMLA, gen_helper_gvec_fmla_nf_s, gen_helper_gvec_fmla_nf_h)
1024	DO_3S_FP_GVEC(VMLS, gen_helper_gvec_fmls_nf_s, gen_helper_gvec_fmls_nf_h)
1025	DO_3S_FP_GVEC(VFMA, gen_helper_gvec_vfma_s, gen_helper_gvec_vfma_h)
1026	DO_3S_FP_GVEC(VFMS, gen_helper_gvec_vfms_s, gen_helper_gvec_vfms_h)
1027	DO_3S_FP_GVEC(VRECPS, gen_helper_gvec_recps_nf_s, gen_helper_gvec_recps_nf_h)
1028	DO_3S_FP_GVEC(VRSQRTS, gen_helper_gvec_rsqrts_nf_s, gen_helper_gvec_rsqrts_nf_h)
1029	DO_3S_FP_GVEC(VPADD, gen_helper_gvec_faddp_s, gen_helper_gvec_faddp_h)
1030	DO_3S_FP_GVEC(VPMAX, gen_helper_gvec_fmaxp_s, gen_helper_gvec_fmaxp_h)
1031	DO_3S_FP_GVEC(VPMIN, gen_helper_gvec_fminp_s, gen_helper_gvec_fminp_h)
1032
1033	WRAP_FP_GVEC(gen_VMAXNM_fp32_3s, FPST_STD, gen_helper_gvec_fmaxnum_s)
1034	WRAP_FP_GVEC(gen_VMAXNM_fp16_3s, FPST_STD_F16, gen_helper_gvec_fmaxnum_h)
1035	WRAP_FP_GVEC(gen_VMINNM_fp32_3s, FPST_STD, gen_helper_gvec_fminnum_s)
1036	WRAP_FP_GVEC(gen_VMINNM_fp16_3s, FPST_STD_F16, gen_helper_gvec_fminnum_h)
1037
1038	static bool trans_VMAXNM_fp_3s(DisasContext s, arg_3same a)
1039	{
1040	if (!arm_dc_feature(s, ARM_FEATURE_V8)) {
1041	return false;
1042	}
1043
1044	if (a->size == MO_16) {
1045	if (!dc_isar_feature(aa32_fp16_arith, s)) {
1046	return false;
1047	}
1048	return do_3same(s, a, gen_VMAXNM_fp16_3s);
1049	}
1050	return do_3same(s, a, gen_VMAXNM_fp32_3s);
1051	}
1052
1053	static bool trans_VMINNM_fp_3s(DisasContext s, arg_3same a)
1054	{
1055	if (!arm_dc_feature(s, ARM_FEATURE_V8)) {
1056	return false;
1057	}
1058
1059	if (a->size == MO_16) {
1060	if (!dc_isar_feature(aa32_fp16_arith, s)) {
1061	return false;
1062	}
1063	return do_3same(s, a, gen_VMINNM_fp16_3s);
1064	}
1065	return do_3same(s, a, gen_VMINNM_fp32_3s);
1066	}
1067
1068	static bool do_vector_2sh(DisasContext s, arg_2reg_shift a, GVecGen2iFn *fn)
1069	{
1070	/* Handle a 2-reg-shift insn which can be vectorized. */
1071	int vec_size = a->q ? 16 : 8;
1072	int rd_ofs = neon_full_reg_offset(a->vd);
1073	int rm_ofs = neon_full_reg_offset(a->vm);
1074
1075	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1076	return false;
1077	}
1078
1079	/* UNDEF accesses to D16-D31 if they don't exist. */
1080	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
1081	return false;
1082	}
1083
1084	if ((a->vm \| a->vd) & a->q) {
1085	return false;
1086	}
1087
1088	if (!neon_access_check(s)) {
1089	return true;
1090	}
1091
1092	fn(a->size, rd_ofs, rm_ofs, a->shift, vec_size, vec_size);
1093	return true;
1094	}
1095
1096	#define DO_2SH(INSN, FUNC) \
1097	static bool trans_##INSN##_2sh(DisasContext s, arg_2reg_shift a) \
1098	{ \
1099	return do_vector_2sh(s, a, FUNC); \
1100	} \
1101
1102	DO_2SH(VSHL, tcg_gen_gvec_shli)
1103	DO_2SH(VSLI, gen_gvec_sli)
1104	DO_2SH(VSRI, gen_gvec_sri)
1105	DO_2SH(VSRA_S, gen_gvec_ssra)
1106	DO_2SH(VSRA_U, gen_gvec_usra)
1107	DO_2SH(VRSHR_S, gen_gvec_srshr)
1108	DO_2SH(VRSHR_U, gen_gvec_urshr)
1109	DO_2SH(VRSRA_S, gen_gvec_srsra)
1110	DO_2SH(VRSRA_U, gen_gvec_ursra)
1111	DO_2SH(VSHR_S, gen_gvec_sshr)
1112	DO_2SH(VSHR_U, gen_gvec_ushr)
1113	DO_2SH(VQSHLU, gen_neon_sqshlui)
1114	DO_2SH(VQSHL_U, gen_neon_uqshli)
1115	DO_2SH(VQSHL_S, gen_neon_sqshli)
1116
1117	static bool do_2shift_narrow_64(DisasContext s, arg_2reg_shift a,
1118	NeonGenTwo64OpFn *shiftfn,
1119	NeonGenOne64OpEnvFn *narrowfn)
1120	{
1121	/* 2-reg-and-shift narrowing-shift operations, size == 3 case */
1122	TCGv_i64 constimm, rm1, rm2, rd;
1123
1124	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1125	return false;
1126	}
1127
1128	/* UNDEF accesses to D16-D31 if they don't exist. */
1129	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
1130	return false;
1131	}
1132
1133	if (a->vm & 1) {
1134	return false;
1135	}
1136
1137	if (!neon_access_check(s)) {
1138	return true;
1139	}
1140
1141	/*
1142	* This is always a right shift, and the shiftfn is always a
1143	* left-shift helper, which thus needs the negated shift count.
1144	*/
1145	constimm = tcg_constant_i64(-a->shift);
1146	rm1 = tcg_temp_new_i64();
1147	rm2 = tcg_temp_new_i64();
1148	rd = tcg_temp_new_i64();
1149
1150	/* Load both inputs first to avoid potential overwrite if rm == rd */
1151	read_neon_element64(rm1, a->vm, 0, MO_64);
1152	read_neon_element64(rm2, a->vm, 1, MO_64);
1153
1154	shiftfn(rm1, rm1, constimm);
1155	narrowfn(rd, tcg_env, rm1);
1156	write_neon_element64(rd, a->vd, 0, MO_32);
1157
1158	shiftfn(rm2, rm2, constimm);
1159	narrowfn(rd, tcg_env, rm2);
1160	write_neon_element64(rd, a->vd, 1, MO_32);
1161
1162	return true;
1163	}
1164
1165	static bool do_2shift_narrow_32(DisasContext s, arg_2reg_shift a,
1166	NeonGenTwoOpFn *shiftfn,
1167	NeonGenOne64OpEnvFn *narrowfn)
1168	{
1169	/* 2-reg-and-shift narrowing-shift operations, size < 3 case */
1170	TCGv_i32 constimm, rm1, rm2, rm3, rm4;
1171	TCGv_i64 rtmp;
1172	uint32_t imm;
1173
1174	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1175	return false;
1176	}
1177
1178	/* UNDEF accesses to D16-D31 if they don't exist. */
1179	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
1180	return false;
1181	}
1182
1183	if (a->vm & 1) {
1184	return false;
1185	}
1186
1187	if (!neon_access_check(s)) {
1188	return true;
1189	}
1190
1191	/*
1192	* This is always a right shift, and the shiftfn is always a
1193	* left-shift helper, which thus needs the negated shift count
1194	* duplicated into each lane of the immediate value.
1195	*/
1196	if (a->size == 1) {
1197	imm = (uint16_t)(-a->shift);
1198	imm \|= imm << 16;
1199	} else {
1200	/* size == 2 */
1201	imm = -a->shift;
1202	}
1203	constimm = tcg_constant_i32(imm);
1204
1205	/* Load all inputs first to avoid potential overwrite */
1206	rm1 = tcg_temp_new_i32();
1207	rm2 = tcg_temp_new_i32();
1208	rm3 = tcg_temp_new_i32();
1209	rm4 = tcg_temp_new_i32();
1210	read_neon_element32(rm1, a->vm, 0, MO_32);
1211	read_neon_element32(rm2, a->vm, 1, MO_32);
1212	read_neon_element32(rm3, a->vm, 2, MO_32);
1213	read_neon_element32(rm4, a->vm, 3, MO_32);
1214	rtmp = tcg_temp_new_i64();
1215
1216	shiftfn(rm1, rm1, constimm);
1217	shiftfn(rm2, rm2, constimm);
1218
1219	tcg_gen_concat_i32_i64(rtmp, rm1, rm2);
1220
1221	narrowfn(rtmp, tcg_env, rtmp);
1222	write_neon_element64(rtmp, a->vd, 0, MO_32);
1223
1224	shiftfn(rm3, rm3, constimm);
1225	shiftfn(rm4, rm4, constimm);
1226
1227	tcg_gen_concat_i32_i64(rtmp, rm3, rm4);
1228
1229	narrowfn(rtmp, tcg_env, rtmp);
1230	write_neon_element64(rtmp, a->vd, 1, MO_32);
1231	return true;
1232	}
1233
1234	#define DO_2SN_64(INSN, FUNC, NARROWFUNC) \
1235	static bool trans_##INSN##_2sh(DisasContext s, arg_2reg_shift a) \
1236	{ \
1237	return do_2shift_narrow_64(s, a, FUNC, NARROWFUNC); \
1238	}
1239	#define DO_2SN_32(INSN, FUNC, NARROWFUNC) \
1240	static bool trans_##INSN##_2sh(DisasContext s, arg_2reg_shift a) \
1241	{ \
1242	return do_2shift_narrow_32(s, a, FUNC, NARROWFUNC); \
1243	}
1244
1245	static void gen_neon_narrow_u32(TCGv_i64 dest, TCGv_ptr env, TCGv_i64 src)
1246	{
1247	tcg_gen_ext32u_i64(dest, src);
1248	}
1249
1250	static void gen_neon_narrow_u16(TCGv_i64 dest, TCGv_ptr env, TCGv_i64 src)
1251	{
1252	gen_helper_neon_narrow_u16(dest, src);
1253	}
1254
1255	static void gen_neon_narrow_u8(TCGv_i64 dest, TCGv_ptr env, TCGv_i64 src)
1256	{
1257	gen_helper_neon_narrow_u8(dest, src);
1258	}
1259
1260	DO_2SN_64(VSHRN_64, gen_ushl_i64, gen_neon_narrow_u32)
1261	DO_2SN_32(VSHRN_32, gen_ushl_i32, gen_neon_narrow_u16)
1262	DO_2SN_32(VSHRN_16, gen_helper_neon_shl_u16, gen_neon_narrow_u8)
1263
1264	DO_2SN_64(VRSHRN_64, gen_helper_neon_rshl_u64, gen_neon_narrow_u32)
1265	DO_2SN_32(VRSHRN_32, gen_helper_neon_rshl_u32, gen_neon_narrow_u16)
1266	DO_2SN_32(VRSHRN_16, gen_helper_neon_rshl_u16, gen_neon_narrow_u8)
1267
1268	DO_2SN_64(VQSHRUN_64, gen_sshl_i64, gen_helper_neon_unarrow_sat32)
1269	DO_2SN_32(VQSHRUN_32, gen_sshl_i32, gen_helper_neon_unarrow_sat16)
1270	DO_2SN_32(VQSHRUN_16, gen_helper_neon_shl_s16, gen_helper_neon_unarrow_sat8)
1271
1272	DO_2SN_64(VQRSHRUN_64, gen_helper_neon_rshl_s64, gen_helper_neon_unarrow_sat32)
1273	DO_2SN_32(VQRSHRUN_32, gen_helper_neon_rshl_s32, gen_helper_neon_unarrow_sat16)
1274	DO_2SN_32(VQRSHRUN_16, gen_helper_neon_rshl_s16, gen_helper_neon_unarrow_sat8)
1275	DO_2SN_64(VQSHRN_S64, gen_sshl_i64, gen_helper_neon_narrow_sat_s32)
1276	DO_2SN_32(VQSHRN_S32, gen_sshl_i32, gen_helper_neon_narrow_sat_s16)
1277	DO_2SN_32(VQSHRN_S16, gen_helper_neon_shl_s16, gen_helper_neon_narrow_sat_s8)
1278
1279	DO_2SN_64(VQRSHRN_S64, gen_helper_neon_rshl_s64, gen_helper_neon_narrow_sat_s32)
1280	DO_2SN_32(VQRSHRN_S32, gen_helper_neon_rshl_s32, gen_helper_neon_narrow_sat_s16)
1281	DO_2SN_32(VQRSHRN_S16, gen_helper_neon_rshl_s16, gen_helper_neon_narrow_sat_s8)
1282
1283	DO_2SN_64(VQSHRN_U64, gen_ushl_i64, gen_helper_neon_narrow_sat_u32)
1284	DO_2SN_32(VQSHRN_U32, gen_ushl_i32, gen_helper_neon_narrow_sat_u16)
1285	DO_2SN_32(VQSHRN_U16, gen_helper_neon_shl_u16, gen_helper_neon_narrow_sat_u8)
1286
1287	DO_2SN_64(VQRSHRN_U64, gen_helper_neon_rshl_u64, gen_helper_neon_narrow_sat_u32)
1288	DO_2SN_32(VQRSHRN_U32, gen_helper_neon_rshl_u32, gen_helper_neon_narrow_sat_u16)
1289	DO_2SN_32(VQRSHRN_U16, gen_helper_neon_rshl_u16, gen_helper_neon_narrow_sat_u8)
1290
1291	static bool do_vshll_2sh(DisasContext s, arg_2reg_shift a,
1292	NeonGenWidenFn *widenfn, bool u)
1293	{
1294	TCGv_i64 tmp;
1295	TCGv_i32 rm0, rm1;
1296	uint64_t widen_mask = 0;
1297
1298	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1299	return false;
1300	}
1301
1302	/* UNDEF accesses to D16-D31 if they don't exist. */
1303	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
1304	return false;
1305	}
1306
1307	if (a->vd & 1) {
1308	return false;
1309	}
1310
1311	if (!neon_access_check(s)) {
1312	return true;
1313	}
1314
1315	/*
1316	* This is a widen-and-shift operation. The shift is always less
1317	* than the width of the source type, so after widening the input
1318	* vector we can simply shift the whole 64-bit widened register,
1319	* and then clear the potential overflow bits resulting from left
1320	* bits of the narrow input appearing as right bits of the left
1321	* neighbour narrow input. Calculate a mask of bits to clear.
1322	*/
1323	if ((a->shift != 0) && (a->size < 2 \|\| u)) {
1324	int esize = 8 << a->size;
1325	widen_mask = MAKE_64BIT_MASK(0, esize);
1326	widen_mask >>= esize - a->shift;
1327	widen_mask = dup_const(a->size + 1, widen_mask);
1328	}
1329
1330	rm0 = tcg_temp_new_i32();
1331	rm1 = tcg_temp_new_i32();
1332	read_neon_element32(rm0, a->vm, 0, MO_32);
1333	read_neon_element32(rm1, a->vm, 1, MO_32);
1334	tmp = tcg_temp_new_i64();
1335
1336	widenfn(tmp, rm0);
1337	if (a->shift != 0) {
1338	tcg_gen_shli_i64(tmp, tmp, a->shift);
1339	tcg_gen_andi_i64(tmp, tmp, ~widen_mask);
1340	}
1341	write_neon_element64(tmp, a->vd, 0, MO_64);
1342
1343	widenfn(tmp, rm1);
1344	if (a->shift != 0) {
1345	tcg_gen_shli_i64(tmp, tmp, a->shift);
1346	tcg_gen_andi_i64(tmp, tmp, ~widen_mask);
1347	}
1348	write_neon_element64(tmp, a->vd, 1, MO_64);
1349	return true;
1350	}
1351
1352	static bool trans_VSHLL_S_2sh(DisasContext s, arg_2reg_shift a)
1353	{
1354	static NeonGenWidenFn * const widenfn[] = {
1355	gen_helper_neon_widen_s8,
1356	gen_helper_neon_widen_s16,
1357	tcg_gen_ext_i32_i64,
1358	};
1359	return do_vshll_2sh(s, a, widenfn[a->size], false);
1360	}
1361
1362	static bool trans_VSHLL_U_2sh(DisasContext s, arg_2reg_shift a)
1363	{
1364	static NeonGenWidenFn * const widenfn[] = {
1365	gen_helper_neon_widen_u8,
1366	gen_helper_neon_widen_u16,
1367	tcg_gen_extu_i32_i64,
1368	};
1369	return do_vshll_2sh(s, a, widenfn[a->size], true);
1370	}
1371
1372	static bool do_fp_2sh(DisasContext s, arg_2reg_shift a,
1373	gen_helper_gvec_2_ptr *fn)
1374	{
1375	/* FP operations in 2-reg-and-shift group */
1376	int vec_size = a->q ? 16 : 8;
1377	int rd_ofs = neon_full_reg_offset(a->vd);
1378	int rm_ofs = neon_full_reg_offset(a->vm);
1379	TCGv_ptr fpst;
1380
1381	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1382	return false;
1383	}
1384
1385	if (a->size == MO_16) {
1386	if (!dc_isar_feature(aa32_fp16_arith, s)) {
1387	return false;
1388	}
1389	}
1390
1391	/* UNDEF accesses to D16-D31 if they don't exist. */
1392	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
1393	return false;
1394	}
1395
1396	if ((a->vm \| a->vd) & a->q) {
1397	return false;
1398	}
1399
1400	if (!neon_access_check(s)) {
1401	return true;
1402	}
1403
1404	fpst = fpstatus_ptr(a->size == MO_16 ? FPST_STD_F16 : FPST_STD);
1405	tcg_gen_gvec_2_ptr(rd_ofs, rm_ofs, fpst, vec_size, vec_size, a->shift, fn);
1406	return true;
1407	}
1408
1409	#define DO_FP_2SH(INSN, FUNC) \
1410	static bool trans_##INSN##_2sh(DisasContext s, arg_2reg_shift a) \
1411	{ \
1412	return do_fp_2sh(s, a, FUNC); \
1413	}
1414
1415	DO_FP_2SH(VCVT_SF, gen_helper_gvec_vcvt_sf)
1416	DO_FP_2SH(VCVT_UF, gen_helper_gvec_vcvt_uf)
1417	DO_FP_2SH(VCVT_FS, gen_helper_gvec_vcvt_rz_fs)
1418	DO_FP_2SH(VCVT_FU, gen_helper_gvec_vcvt_rz_fu)
1419
1420	DO_FP_2SH(VCVT_SH, gen_helper_gvec_vcvt_sh)
1421	DO_FP_2SH(VCVT_UH, gen_helper_gvec_vcvt_uh)
1422	DO_FP_2SH(VCVT_HS, gen_helper_gvec_vcvt_rz_hs)
1423	DO_FP_2SH(VCVT_HU, gen_helper_gvec_vcvt_rz_hu)
1424
1425	static bool do_1reg_imm(DisasContext s, arg_1reg_imm a,
1426	GVecGen2iFn *fn)
1427	{
1428	uint64_t imm;
1429	int reg_ofs, vec_size;
1430
1431	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1432	return false;
1433	}
1434
1435	/* UNDEF accesses to D16-D31 if they don't exist. */
1436	if (!neon_dregs_ok(s, a->vd)) {
1437	return false;
1438	}
1439
1440	if (a->vd & a->q) {
1441	return false;
1442	}
1443
1444	if (!neon_access_check(s)) {
1445	return true;
1446	}
1447
1448	reg_ofs = neon_full_reg_offset(a->vd);
1449	vec_size = a->q ? 16 : 8;
1450	imm = asimd_imm_const(a->imm, a->cmode, a->op);
1451
1452	fn(MO_64, reg_ofs, reg_ofs, imm, vec_size, vec_size);
1453	return true;
1454	}
1455
1456	static void gen_VMOV_1r(unsigned vece, uint32_t dofs, uint32_t aofs,
1457	int64_t c, uint32_t oprsz, uint32_t maxsz)
1458	{
1459	tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, c);
1460	}
1461
1462	static bool trans_Vimm_1r(DisasContext s, arg_1reg_imm a)
1463	{
1464	/* Handle decode of cmode/op here between VORR/VBIC/VMOV */
1465	GVecGen2iFn *fn;
1466
1467	if ((a->cmode & 1) && a->cmode < 12) {
1468	/* for op=1, the imm will be inverted, so BIC becomes AND. */
1469	fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori;
1470	} else {
1471	/* There is one unallocated cmode/op combination in this space */
1472	if (a->cmode == 15 && a->op == 1) {
1473	return false;
1474	}
1475	fn = gen_VMOV_1r;
1476	}
1477	return do_1reg_imm(s, a, fn);
1478	}
1479
1480	static bool do_prewiden_3d(DisasContext s, arg_3diff a,
1481	NeonGenWidenFn *widenfn,
1482	NeonGenTwo64OpFn *opfn,
1483	int src1_mop, int src2_mop)
1484	{
1485	/* 3-regs different lengths, prewidening case (VADDL/VSUBL/VAADW/VSUBW) */
1486	TCGv_i64 rn0_64, rn1_64, rm_64;
1487
1488	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1489	return false;
1490	}
1491
1492	/* UNDEF accesses to D16-D31 if they don't exist. */
1493	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
1494	return false;
1495	}
1496
1497	if (!opfn) {
1498	/* size == 3 case, which is an entirely different insn group */
1499	return false;
1500	}
1501
1502	if ((a->vd & 1) \|\| (src1_mop == MO_UQ && (a->vn & 1))) {
1503	return false;
1504	}
1505
1506	if (!neon_access_check(s)) {
1507	return true;
1508	}
1509
1510	rn0_64 = tcg_temp_new_i64();
1511	rn1_64 = tcg_temp_new_i64();
1512	rm_64 = tcg_temp_new_i64();
1513
1514	if (src1_mop >= 0) {
1515	read_neon_element64(rn0_64, a->vn, 0, src1_mop);
1516	} else {
1517	TCGv_i32 tmp = tcg_temp_new_i32();
1518	read_neon_element32(tmp, a->vn, 0, MO_32);
1519	widenfn(rn0_64, tmp);
1520	}
1521	if (src2_mop >= 0) {
1522	read_neon_element64(rm_64, a->vm, 0, src2_mop);
1523	} else {
1524	TCGv_i32 tmp = tcg_temp_new_i32();
1525	read_neon_element32(tmp, a->vm, 0, MO_32);
1526	widenfn(rm_64, tmp);
1527	}
1528
1529	opfn(rn0_64, rn0_64, rm_64);
1530
1531	/*
1532	* Load second pass inputs before storing the first pass result, to
1533	* avoid incorrect results if a narrow input overlaps with the result.
1534	*/
1535	if (src1_mop >= 0) {
1536	read_neon_element64(rn1_64, a->vn, 1, src1_mop);
1537	} else {
1538	TCGv_i32 tmp = tcg_temp_new_i32();
1539	read_neon_element32(tmp, a->vn, 1, MO_32);
1540	widenfn(rn1_64, tmp);
1541	}
1542	if (src2_mop >= 0) {
1543	read_neon_element64(rm_64, a->vm, 1, src2_mop);
1544	} else {
1545	TCGv_i32 tmp = tcg_temp_new_i32();
1546	read_neon_element32(tmp, a->vm, 1, MO_32);
1547	widenfn(rm_64, tmp);
1548	}
1549
1550	write_neon_element64(rn0_64, a->vd, 0, MO_64);
1551
1552	opfn(rn1_64, rn1_64, rm_64);
1553	write_neon_element64(rn1_64, a->vd, 1, MO_64);
1554
1555	return true;
1556	}
1557
1558	#define DO_PREWIDEN(INSN, S, OP, SRC1WIDE, SIGN) \
1559	static bool trans_##INSN##_3d(DisasContext s, arg_3diff a) \
1560	{ \
1561	static NeonGenWidenFn * const widenfn[] = { \
1562	gen_helper_neon_widen_##S##8, \
1563	gen_helper_neon_widen_##S##16, \
1564	NULL, NULL, \
1565	}; \
1566	static NeonGenTwo64OpFn * const addfn[] = { \
1567	tcg_gen_vec_##OP##16_i64, \
1568	tcg_gen_vec_##OP##32_i64, \
1569	tcg_gen_##OP##_i64, \
1570	NULL, \
1571	}; \
1572	int narrow_mop = a->size == MO_32 ? MO_32 \| SIGN : -1; \
1573	return do_prewiden_3d(s, a, widenfn[a->size], addfn[a->size], \
1574	SRC1WIDE ? MO_UQ : narrow_mop, \
1575	narrow_mop); \
1576	}
1577
1578	DO_PREWIDEN(VADDL_S, s, add, false, MO_SIGN)
1579	DO_PREWIDEN(VADDL_U, u, add, false, 0)
1580	DO_PREWIDEN(VSUBL_S, s, sub, false, MO_SIGN)
1581	DO_PREWIDEN(VSUBL_U, u, sub, false, 0)
1582	DO_PREWIDEN(VADDW_S, s, add, true, MO_SIGN)
1583	DO_PREWIDEN(VADDW_U, u, add, true, 0)
1584	DO_PREWIDEN(VSUBW_S, s, sub, true, MO_SIGN)
1585	DO_PREWIDEN(VSUBW_U, u, sub, true, 0)
1586
1587	static bool do_narrow_3d(DisasContext s, arg_3diff a,
1588	NeonGenTwo64OpFn opfn, NeonGenNarrowFn narrowfn)
1589	{
1590	/* 3-regs different lengths, narrowing (VADDHN/VSUBHN/VRADDHN/VRSUBHN) */
1591	TCGv_i64 rn_64, rm_64;
1592	TCGv_i32 rd0, rd1;
1593
1594	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1595	return false;
1596	}
1597
1598	/* UNDEF accesses to D16-D31 if they don't exist. */
1599	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
1600	return false;
1601	}
1602
1603	if (!opfn \|\| !narrowfn) {
1604	/* size == 3 case, which is an entirely different insn group */
1605	return false;
1606	}
1607
1608	if ((a->vn \| a->vm) & 1) {
1609	return false;
1610	}
1611
1612	if (!neon_access_check(s)) {
1613	return true;
1614	}
1615
1616	rn_64 = tcg_temp_new_i64();
1617	rm_64 = tcg_temp_new_i64();
1618	rd0 = tcg_temp_new_i32();
1619	rd1 = tcg_temp_new_i32();
1620
1621	read_neon_element64(rn_64, a->vn, 0, MO_64);
1622	read_neon_element64(rm_64, a->vm, 0, MO_64);
1623
1624	opfn(rn_64, rn_64, rm_64);
1625
1626	narrowfn(rd0, rn_64);
1627
1628	read_neon_element64(rn_64, a->vn, 1, MO_64);
1629	read_neon_element64(rm_64, a->vm, 1, MO_64);
1630
1631	opfn(rn_64, rn_64, rm_64);
1632
1633	narrowfn(rd1, rn_64);
1634
1635	write_neon_element32(rd0, a->vd, 0, MO_32);
1636	write_neon_element32(rd1, a->vd, 1, MO_32);
1637
1638	return true;
1639	}
1640
1641	#define DO_NARROW_3D(INSN, OP, NARROWTYPE, EXTOP) \
1642	static bool trans_##INSN##_3d(DisasContext s, arg_3diff a) \
1643	{ \
1644	static NeonGenTwo64OpFn * const addfn[] = { \
1645	tcg_gen_vec_##OP##16_i64, \
1646	tcg_gen_vec_##OP##32_i64, \
1647	tcg_gen_##OP##_i64, \
1648	NULL, \
1649	}; \
1650	static NeonGenNarrowFn * const narrowfn[] = { \
1651	gen_helper_neon_##NARROWTYPE##_high_u8, \
1652	gen_helper_neon_##NARROWTYPE##_high_u16, \
1653	EXTOP, \
1654	NULL, \
1655	}; \
1656	return do_narrow_3d(s, a, addfn[a->size], narrowfn[a->size]); \
1657	}
1658
1659	static void gen_narrow_round_high_u32(TCGv_i32 rd, TCGv_i64 rn)
1660	{
1661	tcg_gen_addi_i64(rn, rn, 1u << 31);
1662	tcg_gen_extrh_i64_i32(rd, rn);
1663	}
1664
1665	DO_NARROW_3D(VADDHN, add, narrow, tcg_gen_extrh_i64_i32)
1666	DO_NARROW_3D(VSUBHN, sub, narrow, tcg_gen_extrh_i64_i32)
1667	DO_NARROW_3D(VRADDHN, add, narrow_round, gen_narrow_round_high_u32)
1668	DO_NARROW_3D(VRSUBHN, sub, narrow_round, gen_narrow_round_high_u32)
1669
1670	static bool do_long_3d(DisasContext s, arg_3diff a,
1671	NeonGenTwoOpWidenFn *opfn,
1672	NeonGenTwo64OpFn *accfn)
1673	{
1674	/*
1675	* 3-regs different lengths, long operations.
1676	* These perform an operation on two inputs that returns a double-width
1677	* result, and then possibly perform an accumulation operation of
1678	* that result into the double-width destination.
1679	*/
1680	TCGv_i64 rd0, rd1, tmp;
1681	TCGv_i32 rn, rm;
1682
1683	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1684	return false;
1685	}
1686
1687	/* UNDEF accesses to D16-D31 if they don't exist. */
1688	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
1689	return false;
1690	}
1691
1692	if (!opfn) {
1693	/* size == 3 case, which is an entirely different insn group */
1694	return false;
1695	}
1696
1697	if (a->vd & 1) {
1698	return false;
1699	}
1700
1701	if (!neon_access_check(s)) {
1702	return true;
1703	}
1704
1705	rd0 = tcg_temp_new_i64();
1706	rd1 = tcg_temp_new_i64();
1707
1708	rn = tcg_temp_new_i32();
1709	rm = tcg_temp_new_i32();
1710	read_neon_element32(rn, a->vn, 0, MO_32);
1711	read_neon_element32(rm, a->vm, 0, MO_32);
1712	opfn(rd0, rn, rm);
1713
1714	read_neon_element32(rn, a->vn, 1, MO_32);
1715	read_neon_element32(rm, a->vm, 1, MO_32);
1716	opfn(rd1, rn, rm);
1717
1718	/* Don't store results until after all loads: they might overlap */
1719	if (accfn) {
1720	tmp = tcg_temp_new_i64();
1721	read_neon_element64(tmp, a->vd, 0, MO_64);
1722	accfn(rd0, tmp, rd0);
1723	read_neon_element64(tmp, a->vd, 1, MO_64);
1724	accfn(rd1, tmp, rd1);
1725	}
1726
1727	write_neon_element64(rd0, a->vd, 0, MO_64);
1728	write_neon_element64(rd1, a->vd, 1, MO_64);
1729
1730	return true;
1731	}
1732
1733	static bool trans_VABDL_S_3d(DisasContext s, arg_3diff a)
1734	{
1735	static NeonGenTwoOpWidenFn * const opfn[] = {
1736	gen_helper_neon_abdl_s16,
1737	gen_helper_neon_abdl_s32,
1738	gen_helper_neon_abdl_s64,
1739	NULL,
1740	};
1741
1742	return do_long_3d(s, a, opfn[a->size], NULL);
1743	}
1744
1745	static bool trans_VABDL_U_3d(DisasContext s, arg_3diff a)
1746	{
1747	static NeonGenTwoOpWidenFn * const opfn[] = {
1748	gen_helper_neon_abdl_u16,
1749	gen_helper_neon_abdl_u32,
1750	gen_helper_neon_abdl_u64,
1751	NULL,
1752	};
1753
1754	return do_long_3d(s, a, opfn[a->size], NULL);
1755	}
1756
1757	static bool trans_VABAL_S_3d(DisasContext s, arg_3diff a)
1758	{
1759	static NeonGenTwoOpWidenFn * const opfn[] = {
1760	gen_helper_neon_abdl_s16,
1761	gen_helper_neon_abdl_s32,
1762	gen_helper_neon_abdl_s64,
1763	NULL,
1764	};
1765	static NeonGenTwo64OpFn * const addfn[] = {
1766	tcg_gen_vec_add16_i64,
1767	tcg_gen_vec_add32_i64,
1768	tcg_gen_add_i64,
1769	NULL,
1770	};
1771
1772	return do_long_3d(s, a, opfn[a->size], addfn[a->size]);
1773	}
1774
1775	static bool trans_VABAL_U_3d(DisasContext s, arg_3diff a)
1776	{
1777	static NeonGenTwoOpWidenFn * const opfn[] = {
1778	gen_helper_neon_abdl_u16,
1779	gen_helper_neon_abdl_u32,
1780	gen_helper_neon_abdl_u64,
1781	NULL,
1782	};
1783	static NeonGenTwo64OpFn * const addfn[] = {
1784	tcg_gen_vec_add16_i64,
1785	tcg_gen_vec_add32_i64,
1786	tcg_gen_add_i64,
1787	NULL,
1788	};
1789
1790	return do_long_3d(s, a, opfn[a->size], addfn[a->size]);
1791	}
1792
1793	static void gen_mull_s32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm)
1794	{
1795	TCGv_i32 lo = tcg_temp_new_i32();
1796	TCGv_i32 hi = tcg_temp_new_i32();
1797
1798	tcg_gen_muls2_i32(lo, hi, rn, rm);
1799	tcg_gen_concat_i32_i64(rd, lo, hi);
1800	}
1801
1802	static void gen_mull_u32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm)
1803	{
1804	TCGv_i32 lo = tcg_temp_new_i32();
1805	TCGv_i32 hi = tcg_temp_new_i32();
1806
1807	tcg_gen_mulu2_i32(lo, hi, rn, rm);
1808	tcg_gen_concat_i32_i64(rd, lo, hi);
1809	}
1810
1811	static bool trans_VMULL_S_3d(DisasContext s, arg_3diff a)
1812	{
1813	static NeonGenTwoOpWidenFn * const opfn[] = {
1814	gen_helper_neon_mull_s8,
1815	gen_helper_neon_mull_s16,
1816	gen_mull_s32,
1817	NULL,
1818	};
1819
1820	return do_long_3d(s, a, opfn[a->size], NULL);
1821	}
1822
1823	static bool trans_VMULL_U_3d(DisasContext s, arg_3diff a)
1824	{
1825	static NeonGenTwoOpWidenFn * const opfn[] = {
1826	gen_helper_neon_mull_u8,
1827	gen_helper_neon_mull_u16,
1828	gen_mull_u32,
1829	NULL,
1830	};
1831
1832	return do_long_3d(s, a, opfn[a->size], NULL);
1833	}
1834
1835	#define DO_VMLAL(INSN,MULL,ACC) \
1836	static bool trans_##INSN##_3d(DisasContext s, arg_3diff a) \
1837	{ \
1838	static NeonGenTwoOpWidenFn * const opfn[] = { \
1839	gen_helper_neon_##MULL##8, \
1840	gen_helper_neon_##MULL##16, \
1841	gen_##MULL##32, \
1842	NULL, \
1843	}; \
1844	static NeonGenTwo64OpFn * const accfn[] = { \
1845	tcg_gen_vec_##ACC##16_i64, \
1846	tcg_gen_vec_##ACC##32_i64, \
1847	tcg_gen_##ACC##_i64, \
1848	NULL, \
1849	}; \
1850	return do_long_3d(s, a, opfn[a->size], accfn[a->size]); \
1851	}
1852
1853	DO_VMLAL(VMLAL_S,mull_s,add)
1854	DO_VMLAL(VMLAL_U,mull_u,add)
1855	DO_VMLAL(VMLSL_S,mull_s,sub)
1856	DO_VMLAL(VMLSL_U,mull_u,sub)
1857
1858	static void gen_VQDMULL_16(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm)
1859	{
1860	gen_helper_neon_mull_s16(rd, rn, rm);
1861	gen_helper_neon_addl_saturate_s32(rd, tcg_env, rd, rd);
1862	}
1863
1864	static void gen_VQDMULL_32(TCGv_i64 rd, TCGv_i32 rn, TCGv_i32 rm)
1865	{
1866	gen_mull_s32(rd, rn, rm);
1867	gen_helper_neon_addl_saturate_s64(rd, tcg_env, rd, rd);
1868	}
1869
1870	static bool trans_VQDMULL_3d(DisasContext s, arg_3diff a)
1871	{
1872	static NeonGenTwoOpWidenFn * const opfn[] = {
1873	NULL,
1874	gen_VQDMULL_16,
1875	gen_VQDMULL_32,
1876	NULL,
1877	};
1878
1879	return do_long_3d(s, a, opfn[a->size], NULL);
1880	}
1881
1882	static void gen_VQDMLAL_acc_16(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm)
1883	{
1884	gen_helper_neon_addl_saturate_s32(rd, tcg_env, rn, rm);
1885	}
1886
1887	static void gen_VQDMLAL_acc_32(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm)
1888	{
1889	gen_helper_neon_addl_saturate_s64(rd, tcg_env, rn, rm);
1890	}
1891
1892	static bool trans_VQDMLAL_3d(DisasContext s, arg_3diff a)
1893	{
1894	static NeonGenTwoOpWidenFn * const opfn[] = {
1895	NULL,
1896	gen_VQDMULL_16,
1897	gen_VQDMULL_32,
1898	NULL,
1899	};
1900	static NeonGenTwo64OpFn * const accfn[] = {
1901	NULL,
1902	gen_VQDMLAL_acc_16,
1903	gen_VQDMLAL_acc_32,
1904	NULL,
1905	};
1906
1907	return do_long_3d(s, a, opfn[a->size], accfn[a->size]);
1908	}
1909
1910	static void gen_VQDMLSL_acc_16(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm)
1911	{
1912	gen_helper_neon_negl_u32(rm, rm);
1913	gen_helper_neon_addl_saturate_s32(rd, tcg_env, rn, rm);
1914	}
1915
1916	static void gen_VQDMLSL_acc_32(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm)
1917	{
1918	tcg_gen_neg_i64(rm, rm);
1919	gen_helper_neon_addl_saturate_s64(rd, tcg_env, rn, rm);
1920	}
1921
1922	static bool trans_VQDMLSL_3d(DisasContext s, arg_3diff a)
1923	{
1924	static NeonGenTwoOpWidenFn * const opfn[] = {
1925	NULL,
1926	gen_VQDMULL_16,
1927	gen_VQDMULL_32,
1928	NULL,
1929	};
1930	static NeonGenTwo64OpFn * const accfn[] = {
1931	NULL,
1932	gen_VQDMLSL_acc_16,
1933	gen_VQDMLSL_acc_32,
1934	NULL,
1935	};
1936
1937	return do_long_3d(s, a, opfn[a->size], accfn[a->size]);
1938	}
1939
1940	static bool trans_VMULL_P_3d(DisasContext s, arg_3diff a)
1941	{
1942	gen_helper_gvec_3 *fn_gvec;
1943
1944	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
1945	return false;
1946	}
1947
1948	/* UNDEF accesses to D16-D31 if they don't exist. */
1949	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
1950	return false;
1951	}
1952
1953	if (a->vd & 1) {
1954	return false;
1955	}
1956
1957	switch (a->size) {
1958	case 0:
1959	fn_gvec = gen_helper_neon_pmull_h;
1960	break;
1961	case 2:
1962	if (!dc_isar_feature(aa32_pmull, s)) {
1963	return false;
1964	}
1965	fn_gvec = gen_helper_gvec_pmull_q;
1966	break;
1967	default:
1968	return false;
1969	}
1970
1971	if (!neon_access_check(s)) {
1972	return true;
1973	}
1974
1975	tcg_gen_gvec_3_ool(neon_full_reg_offset(a->vd),
1976	neon_full_reg_offset(a->vn),
1977	neon_full_reg_offset(a->vm),
1978	16, 16, 0, fn_gvec);
1979	return true;
1980	}
1981
1982	static void gen_neon_dup_low16(TCGv_i32 var)
1983	{
1984	TCGv_i32 tmp = tcg_temp_new_i32();
1985	tcg_gen_ext16u_i32(var, var);
1986	tcg_gen_shli_i32(tmp, var, 16);
1987	tcg_gen_or_i32(var, var, tmp);
1988	}
1989
1990	static void gen_neon_dup_high16(TCGv_i32 var)
1991	{
1992	TCGv_i32 tmp = tcg_temp_new_i32();
1993	tcg_gen_andi_i32(var, var, 0xffff0000);
1994	tcg_gen_shri_i32(tmp, var, 16);
1995	tcg_gen_or_i32(var, var, tmp);
1996	}
1997
1998	static inline TCGv_i32 neon_get_scalar(int size, int reg)
1999	{
2000	TCGv_i32 tmp = tcg_temp_new_i32();
2001	if (size == MO_16) {
2002	read_neon_element32(tmp, reg & 7, reg >> 4, MO_32);
2003	if (reg & 8) {
2004	gen_neon_dup_high16(tmp);
2005	} else {
2006	gen_neon_dup_low16(tmp);
2007	}
2008	} else {
2009	read_neon_element32(tmp, reg & 15, reg >> 4, MO_32);
2010	}
2011	return tmp;
2012	}
2013
2014	static bool do_2scalar(DisasContext s, arg_2scalar a,
2015	NeonGenTwoOpFn opfn, NeonGenTwoOpFn accfn)
2016	{
2017	/*
2018	* Two registers and a scalar: perform an operation between
2019	* the input elements and the scalar, and then possibly
2020	* perform an accumulation operation of that result into the
2021	* destination.
2022	*/
2023	TCGv_i32 scalar, tmp;
2024	int pass;
2025
2026	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2027	return false;
2028	}
2029
2030	/* UNDEF accesses to D16-D31 if they don't exist. */
2031	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
2032	return false;
2033	}
2034
2035	if (!opfn) {
2036	/* Bad size (including size == 3, which is a different insn group) */
2037	return false;
2038	}
2039
2040	if (a->q && ((a->vd \| a->vn) & 1)) {
2041	return false;
2042	}
2043
2044	if (!neon_access_check(s)) {
2045	return true;
2046	}
2047
2048	scalar = neon_get_scalar(a->size, a->vm);
2049	tmp = tcg_temp_new_i32();
2050
2051	for (pass = 0; pass < (a->q ? 4 : 2); pass++) {
2052	read_neon_element32(tmp, a->vn, pass, MO_32);
2053	opfn(tmp, tmp, scalar);
2054	if (accfn) {
2055	TCGv_i32 rd = tcg_temp_new_i32();
2056	read_neon_element32(rd, a->vd, pass, MO_32);
2057	accfn(tmp, rd, tmp);
2058	}
2059	write_neon_element32(tmp, a->vd, pass, MO_32);
2060	}
2061	return true;
2062	}
2063
2064	static bool trans_VMUL_2sc(DisasContext s, arg_2scalar a)
2065	{
2066	static NeonGenTwoOpFn * const opfn[] = {
2067	NULL,
2068	gen_helper_neon_mul_u16,
2069	tcg_gen_mul_i32,
2070	NULL,
2071	};
2072
2073	return do_2scalar(s, a, opfn[a->size], NULL);
2074	}
2075
2076	static bool trans_VMLA_2sc(DisasContext s, arg_2scalar a)
2077	{
2078	static NeonGenTwoOpFn * const opfn[] = {
2079	NULL,
2080	gen_helper_neon_mul_u16,
2081	tcg_gen_mul_i32,
2082	NULL,
2083	};
2084	static NeonGenTwoOpFn * const accfn[] = {
2085	NULL,
2086	gen_helper_neon_add_u16,
2087	tcg_gen_add_i32,
2088	NULL,
2089	};
2090
2091	return do_2scalar(s, a, opfn[a->size], accfn[a->size]);
2092	}
2093
2094	static bool trans_VMLS_2sc(DisasContext s, arg_2scalar a)
2095	{
2096	static NeonGenTwoOpFn * const opfn[] = {
2097	NULL,
2098	gen_helper_neon_mul_u16,
2099	tcg_gen_mul_i32,
2100	NULL,
2101	};
2102	static NeonGenTwoOpFn * const accfn[] = {
2103	NULL,
2104	gen_helper_neon_sub_u16,
2105	tcg_gen_sub_i32,
2106	NULL,
2107	};
2108
2109	return do_2scalar(s, a, opfn[a->size], accfn[a->size]);
2110	}
2111
2112	static bool do_2scalar_fp_vec(DisasContext s, arg_2scalar a,
2113	gen_helper_gvec_3_ptr *fn)
2114	{
2115	/* Two registers and a scalar, using gvec */
2116	int vec_size = a->q ? 16 : 8;
2117	int rd_ofs = neon_full_reg_offset(a->vd);
2118	int rn_ofs = neon_full_reg_offset(a->vn);
2119	int rm_ofs;
2120	int idx;
2121	TCGv_ptr fpstatus;
2122
2123	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2124	return false;
2125	}
2126
2127	/* UNDEF accesses to D16-D31 if they don't exist. */
2128	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
2129	return false;
2130	}
2131
2132	if (!fn) {
2133	/* Bad size (including size == 3, which is a different insn group) */
2134	return false;
2135	}
2136
2137	if (a->q && ((a->vd \| a->vn) & 1)) {
2138	return false;
2139	}
2140
2141	if (!neon_access_check(s)) {
2142	return true;
2143	}
2144
2145	/* a->vm is M:Vm, which encodes both register and index */
2146	idx = extract32(a->vm, a->size + 2, 2);
2147	a->vm = extract32(a->vm, 0, a->size + 2);
2148	rm_ofs = neon_full_reg_offset(a->vm);
2149
2150	fpstatus = fpstatus_ptr(a->size == 1 ? FPST_STD_F16 : FPST_STD);
2151	tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, fpstatus,
2152	vec_size, vec_size, idx, fn);
2153	return true;
2154	}
2155
2156	#define DO_VMUL_F_2sc(NAME, FUNC) \
2157	static bool trans_##NAME##_F_2sc(DisasContext s, arg_2scalar a) \
2158	{ \
2159	static gen_helper_gvec_3_ptr * const opfn[] = { \
2160	NULL, \
2161	gen_helper_##FUNC##_h, \
2162	gen_helper_##FUNC##_s, \
2163	NULL, \
2164	}; \
2165	if (a->size == MO_16 && !dc_isar_feature(aa32_fp16_arith, s)) { \
2166	return false; \
2167	} \
2168	return do_2scalar_fp_vec(s, a, opfn[a->size]); \
2169	}
2170
2171	DO_VMUL_F_2sc(VMUL, gvec_fmul_idx)
2172	DO_VMUL_F_2sc(VMLA, gvec_fmla_nf_idx)
2173	DO_VMUL_F_2sc(VMLS, gvec_fmls_nf_idx)
2174
2175	WRAP_ENV_FN(gen_VQDMULH_16, gen_helper_neon_qdmulh_s16)
2176	WRAP_ENV_FN(gen_VQDMULH_32, gen_helper_neon_qdmulh_s32)
2177	WRAP_ENV_FN(gen_VQRDMULH_16, gen_helper_neon_qrdmulh_s16)
2178	WRAP_ENV_FN(gen_VQRDMULH_32, gen_helper_neon_qrdmulh_s32)
2179
2180	static bool trans_VQDMULH_2sc(DisasContext s, arg_2scalar a)
2181	{
2182	static NeonGenTwoOpFn * const opfn[] = {
2183	NULL,
2184	gen_VQDMULH_16,
2185	gen_VQDMULH_32,
2186	NULL,
2187	};
2188
2189	return do_2scalar(s, a, opfn[a->size], NULL);
2190	}
2191
2192	static bool trans_VQRDMULH_2sc(DisasContext s, arg_2scalar a)
2193	{
2194	static NeonGenTwoOpFn * const opfn[] = {
2195	NULL,
2196	gen_VQRDMULH_16,
2197	gen_VQRDMULH_32,
2198	NULL,
2199	};
2200
2201	return do_2scalar(s, a, opfn[a->size], NULL);
2202	}
2203
2204	static bool do_vqrdmlah_2sc(DisasContext s, arg_2scalar a,
2205	NeonGenThreeOpEnvFn *opfn)
2206	{
2207	/*
2208	* VQRDMLAH/VQRDMLSH: this is like do_2scalar, but the opfn
2209	* performs a kind of fused op-then-accumulate using a helper
2210	* function that takes all of rd, rn and the scalar at once.
2211	*/
2212	TCGv_i32 scalar, rn, rd;
2213	int pass;
2214
2215	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2216	return false;
2217	}
2218
2219	if (!dc_isar_feature(aa32_rdm, s)) {
2220	return false;
2221	}
2222
2223	/* UNDEF accesses to D16-D31 if they don't exist. */
2224	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
2225	return false;
2226	}
2227
2228	if (!opfn) {
2229	/* Bad size (including size == 3, which is a different insn group) */
2230	return false;
2231	}
2232
2233	if (a->q && ((a->vd \| a->vn) & 1)) {
2234	return false;
2235	}
2236
2237	if (!neon_access_check(s)) {
2238	return true;
2239	}
2240
2241	scalar = neon_get_scalar(a->size, a->vm);
2242	rn = tcg_temp_new_i32();
2243	rd = tcg_temp_new_i32();
2244
2245	for (pass = 0; pass < (a->q ? 4 : 2); pass++) {
2246	read_neon_element32(rn, a->vn, pass, MO_32);
2247	read_neon_element32(rd, a->vd, pass, MO_32);
2248	opfn(rd, tcg_env, rn, scalar, rd);
2249	write_neon_element32(rd, a->vd, pass, MO_32);
2250	}
2251	return true;
2252	}
2253
2254	static bool trans_VQRDMLAH_2sc(DisasContext s, arg_2scalar a)
2255	{
2256	static NeonGenThreeOpEnvFn *opfn[] = {
2257	NULL,
2258	gen_helper_neon_qrdmlah_s16,
2259	gen_helper_neon_qrdmlah_s32,
2260	NULL,
2261	};
2262	return do_vqrdmlah_2sc(s, a, opfn[a->size]);
2263	}
2264
2265	static bool trans_VQRDMLSH_2sc(DisasContext s, arg_2scalar a)
2266	{
2267	static NeonGenThreeOpEnvFn *opfn[] = {
2268	NULL,
2269	gen_helper_neon_qrdmlsh_s16,
2270	gen_helper_neon_qrdmlsh_s32,
2271	NULL,
2272	};
2273	return do_vqrdmlah_2sc(s, a, opfn[a->size]);
2274	}
2275
2276	static bool do_2scalar_long(DisasContext s, arg_2scalar a,
2277	NeonGenTwoOpWidenFn *opfn,
2278	NeonGenTwo64OpFn *accfn)
2279	{
2280	/*
2281	* Two registers and a scalar, long operations: perform an
2282	* operation on the input elements and the scalar which produces
2283	* a double-width result, and then possibly perform an accumulation
2284	* operation of that result into the destination.
2285	*/
2286	TCGv_i32 scalar, rn;
2287	TCGv_i64 rn0_64, rn1_64;
2288
2289	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2290	return false;
2291	}
2292
2293	/* UNDEF accesses to D16-D31 if they don't exist. */
2294	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
2295	return false;
2296	}
2297
2298	if (!opfn) {
2299	/* Bad size (including size == 3, which is a different insn group) */
2300	return false;
2301	}
2302
2303	if (a->vd & 1) {
2304	return false;
2305	}
2306
2307	if (!neon_access_check(s)) {
2308	return true;
2309	}
2310
2311	scalar = neon_get_scalar(a->size, a->vm);
2312
2313	/* Load all inputs before writing any outputs, in case of overlap */
2314	rn = tcg_temp_new_i32();
2315	read_neon_element32(rn, a->vn, 0, MO_32);
2316	rn0_64 = tcg_temp_new_i64();
2317	opfn(rn0_64, rn, scalar);
2318
2319	read_neon_element32(rn, a->vn, 1, MO_32);
2320	rn1_64 = tcg_temp_new_i64();
2321	opfn(rn1_64, rn, scalar);
2322
2323	if (accfn) {
2324	TCGv_i64 t64 = tcg_temp_new_i64();
2325	read_neon_element64(t64, a->vd, 0, MO_64);
2326	accfn(rn0_64, t64, rn0_64);
2327	read_neon_element64(t64, a->vd, 1, MO_64);
2328	accfn(rn1_64, t64, rn1_64);
2329	}
2330
2331	write_neon_element64(rn0_64, a->vd, 0, MO_64);
2332	write_neon_element64(rn1_64, a->vd, 1, MO_64);
2333	return true;
2334	}
2335
2336	static bool trans_VMULL_S_2sc(DisasContext s, arg_2scalar a)
2337	{
2338	static NeonGenTwoOpWidenFn * const opfn[] = {
2339	NULL,
2340	gen_helper_neon_mull_s16,
2341	gen_mull_s32,
2342	NULL,
2343	};
2344
2345	return do_2scalar_long(s, a, opfn[a->size], NULL);
2346	}
2347
2348	static bool trans_VMULL_U_2sc(DisasContext s, arg_2scalar a)
2349	{
2350	static NeonGenTwoOpWidenFn * const opfn[] = {
2351	NULL,
2352	gen_helper_neon_mull_u16,
2353	gen_mull_u32,
2354	NULL,
2355	};
2356
2357	return do_2scalar_long(s, a, opfn[a->size], NULL);
2358	}
2359
2360	#define DO_VMLAL_2SC(INSN, MULL, ACC) \
2361	static bool trans_##INSN##_2sc(DisasContext s, arg_2scalar a) \
2362	{ \
2363	static NeonGenTwoOpWidenFn * const opfn[] = { \
2364	NULL, \
2365	gen_helper_neon_##MULL##16, \
2366	gen_##MULL##32, \
2367	NULL, \
2368	}; \
2369	static NeonGenTwo64OpFn * const accfn[] = { \
2370	NULL, \
2371	tcg_gen_vec_##ACC##32_i64, \
2372	tcg_gen_##ACC##_i64, \
2373	NULL, \
2374	}; \
2375	return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]); \
2376	}
2377
2378	DO_VMLAL_2SC(VMLAL_S, mull_s, add)
2379	DO_VMLAL_2SC(VMLAL_U, mull_u, add)
2380	DO_VMLAL_2SC(VMLSL_S, mull_s, sub)
2381	DO_VMLAL_2SC(VMLSL_U, mull_u, sub)
2382
2383	static bool trans_VQDMULL_2sc(DisasContext s, arg_2scalar a)
2384	{
2385	static NeonGenTwoOpWidenFn * const opfn[] = {
2386	NULL,
2387	gen_VQDMULL_16,
2388	gen_VQDMULL_32,
2389	NULL,
2390	};
2391
2392	return do_2scalar_long(s, a, opfn[a->size], NULL);
2393	}
2394
2395	static bool trans_VQDMLAL_2sc(DisasContext s, arg_2scalar a)
2396	{
2397	static NeonGenTwoOpWidenFn * const opfn[] = {
2398	NULL,
2399	gen_VQDMULL_16,
2400	gen_VQDMULL_32,
2401	NULL,
2402	};
2403	static NeonGenTwo64OpFn * const accfn[] = {
2404	NULL,
2405	gen_VQDMLAL_acc_16,
2406	gen_VQDMLAL_acc_32,
2407	NULL,
2408	};
2409
2410	return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]);
2411	}
2412
2413	static bool trans_VQDMLSL_2sc(DisasContext s, arg_2scalar a)
2414	{
2415	static NeonGenTwoOpWidenFn * const opfn[] = {
2416	NULL,
2417	gen_VQDMULL_16,
2418	gen_VQDMULL_32,
2419	NULL,
2420	};
2421	static NeonGenTwo64OpFn * const accfn[] = {
2422	NULL,
2423	gen_VQDMLSL_acc_16,
2424	gen_VQDMLSL_acc_32,
2425	NULL,
2426	};
2427
2428	return do_2scalar_long(s, a, opfn[a->size], accfn[a->size]);
2429	}
2430
2431	static bool trans_VEXT(DisasContext s, arg_VEXT a)
2432	{
2433	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2434	return false;
2435	}
2436
2437	/* UNDEF accesses to D16-D31 if they don't exist. */
2438	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
2439	return false;
2440	}
2441
2442	if ((a->vn \| a->vm \| a->vd) & a->q) {
2443	return false;
2444	}
2445
2446	if (a->imm > 7 && !a->q) {
2447	return false;
2448	}
2449
2450	if (!neon_access_check(s)) {
2451	return true;
2452	}
2453
2454	if (!a->q) {
2455	/* Extract 64 bits from <Vm:Vn> */
2456	TCGv_i64 left, right, dest;
2457
2458	left = tcg_temp_new_i64();
2459	right = tcg_temp_new_i64();
2460	dest = tcg_temp_new_i64();
2461
2462	read_neon_element64(right, a->vn, 0, MO_64);
2463	read_neon_element64(left, a->vm, 0, MO_64);
2464	tcg_gen_extract2_i64(dest, right, left, a->imm * 8);
2465	write_neon_element64(dest, a->vd, 0, MO_64);
2466	} else {
2467	/* Extract 128 bits from <Vm+1:Vm:Vn+1:Vn> */
2468	TCGv_i64 left, middle, right, destleft, destright;
2469
2470	left = tcg_temp_new_i64();
2471	middle = tcg_temp_new_i64();
2472	right = tcg_temp_new_i64();
2473	destleft = tcg_temp_new_i64();
2474	destright = tcg_temp_new_i64();
2475
2476	if (a->imm < 8) {
2477	read_neon_element64(right, a->vn, 0, MO_64);
2478	read_neon_element64(middle, a->vn, 1, MO_64);
2479	tcg_gen_extract2_i64(destright, right, middle, a->imm * 8);
2480	read_neon_element64(left, a->vm, 0, MO_64);
2481	tcg_gen_extract2_i64(destleft, middle, left, a->imm * 8);
2482	} else {
2483	read_neon_element64(right, a->vn, 1, MO_64);
2484	read_neon_element64(middle, a->vm, 0, MO_64);
2485	tcg_gen_extract2_i64(destright, right, middle, (a->imm - 8) * 8);
2486	read_neon_element64(left, a->vm, 1, MO_64);
2487	tcg_gen_extract2_i64(destleft, middle, left, (a->imm - 8) * 8);
2488	}
2489
2490	write_neon_element64(destright, a->vd, 0, MO_64);
2491	write_neon_element64(destleft, a->vd, 1, MO_64);
2492	}
2493	return true;
2494	}
2495
2496	static bool trans_VTBL(DisasContext s, arg_VTBL a)
2497	{
2498	TCGv_i64 val, def;
2499	TCGv_i32 desc;
2500
2501	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2502	return false;
2503	}
2504
2505	/* UNDEF accesses to D16-D31 if they don't exist. */
2506	if (!neon_dregs_ok(s, a->vd \| a->vn \| a->vm)) {
2507	return false;
2508	}
2509
2510	if ((a->vn + a->len + 1) > 32) {
2511	/*
2512	* This is UNPREDICTABLE; we choose to UNDEF to avoid the
2513	* helper function running off the end of the register file.
2514	*/
2515	return false;
2516	}
2517
2518	if (!neon_access_check(s)) {
2519	return true;
2520	}
2521
2522	desc = tcg_constant_i32((a->vn << 2) \| a->len);
2523	def = tcg_temp_new_i64();
2524	if (a->op) {
2525	read_neon_element64(def, a->vd, 0, MO_64);
2526	} else {
2527	tcg_gen_movi_i64(def, 0);
2528	}
2529	val = tcg_temp_new_i64();
2530	read_neon_element64(val, a->vm, 0, MO_64);
2531
2532	gen_helper_neon_tbl(val, tcg_env, desc, val, def);
2533	write_neon_element64(val, a->vd, 0, MO_64);
2534	return true;
2535	}
2536
2537	static bool trans_VDUP_scalar(DisasContext s, arg_VDUP_scalar a)
2538	{
2539	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2540	return false;
2541	}
2542
2543	/* UNDEF accesses to D16-D31 if they don't exist. */
2544	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
2545	return false;
2546	}
2547
2548	if (a->vd & a->q) {
2549	return false;
2550	}
2551
2552	if (!neon_access_check(s)) {
2553	return true;
2554	}
2555
2556	tcg_gen_gvec_dup_mem(a->size, neon_full_reg_offset(a->vd),
2557	neon_element_offset(a->vm, a->index, a->size),
2558	a->q ? 16 : 8, a->q ? 16 : 8);
2559	return true;
2560	}
2561
2562	typedef void ZipFn(TCGv_ptr, TCGv_ptr);
2563
2564	static bool do_zip_uzp(DisasContext s, arg_2misc a,
2565	ZipFn *fn)
2566	{
2567	TCGv_ptr pd, pm;
2568
2569	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2570	return false;
2571	}
2572
2573	/* UNDEF accesses to D16-D31 if they don't exist. */
2574	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
2575	return false;
2576	}
2577
2578	if ((a->vd \| a->vm) & a->q) {
2579	return false;
2580	}
2581
2582	if (!fn) {
2583	/* Bad size or size/q combination */
2584	return false;
2585	}
2586
2587	if (!neon_access_check(s)) {
2588	return true;
2589	}
2590
2591	pd = vfp_reg_ptr(true, a->vd);
2592	pm = vfp_reg_ptr(true, a->vm);
2593	fn(pd, pm);
2594	return true;
2595	}
2596
2597	static bool trans_VUZP(DisasContext s, arg_2misc a)
2598	{
2599	static ZipFn * const fn[2][4] = {
2600	{
2601	gen_helper_neon_unzip8,
2602	gen_helper_neon_unzip16,
2603	NULL,
2604	NULL,
2605	}, {
2606	gen_helper_neon_qunzip8,
2607	gen_helper_neon_qunzip16,
2608	gen_helper_neon_qunzip32,
2609	NULL,
2610	}
2611	};
2612	return do_zip_uzp(s, a, fn[a->q][a->size]);
2613	}
2614
2615	static bool trans_VZIP(DisasContext s, arg_2misc a)
2616	{
2617	static ZipFn * const fn[2][4] = {
2618	{
2619	gen_helper_neon_zip8,
2620	gen_helper_neon_zip16,
2621	NULL,
2622	NULL,
2623	}, {
2624	gen_helper_neon_qzip8,
2625	gen_helper_neon_qzip16,
2626	gen_helper_neon_qzip32,
2627	NULL,
2628	}
2629	};
2630	return do_zip_uzp(s, a, fn[a->q][a->size]);
2631	}
2632
2633	static bool do_vmovn(DisasContext s, arg_2misc a,
2634	NeonGenOne64OpEnvFn *narrowfn)
2635	{
2636	TCGv_i64 rm, rd0, rd1;
2637
2638	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2639	return false;
2640	}
2641
2642	/* UNDEF accesses to D16-D31 if they don't exist. */
2643	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
2644	return false;
2645	}
2646
2647	if (a->vm & 1) {
2648	return false;
2649	}
2650
2651	if (!narrowfn) {
2652	return false;
2653	}
2654
2655	if (!neon_access_check(s)) {
2656	return true;
2657	}
2658
2659	rm = tcg_temp_new_i64();
2660	rd0 = tcg_temp_new_i64();
2661	rd1 = tcg_temp_new_i64();
2662
2663	read_neon_element64(rm, a->vm, 0, MO_64);
2664	narrowfn(rd0, tcg_env, rm);
2665	read_neon_element64(rm, a->vm, 1, MO_64);
2666	narrowfn(rd1, tcg_env, rm);
2667	write_neon_element64(rd0, a->vd, 0, MO_32);
2668	write_neon_element64(rd1, a->vd, 1, MO_32);
2669	return true;
2670	}
2671
2672	#define DO_VMOVN(INSN, FUNC) \
2673	static bool trans_##INSN(DisasContext s, arg_2misc a) \
2674	{ \
2675	static NeonGenOne64OpEnvFn * const narrowfn[] = { \
2676	FUNC##8, \
2677	FUNC##16, \
2678	FUNC##32, \
2679	NULL, \
2680	}; \
2681	return do_vmovn(s, a, narrowfn[a->size]); \
2682	}
2683
2684	DO_VMOVN(VMOVN, gen_neon_narrow_u)
2685	DO_VMOVN(VQMOVUN, gen_helper_neon_unarrow_sat)
2686	DO_VMOVN(VQMOVN_S, gen_helper_neon_narrow_sat_s)
2687	DO_VMOVN(VQMOVN_U, gen_helper_neon_narrow_sat_u)
2688
2689	static bool trans_VSHLL(DisasContext s, arg_2misc a)
2690	{
2691	TCGv_i32 rm0, rm1;
2692	TCGv_i64 rd;
2693	static NeonGenWidenFn * const widenfns[] = {
2694	gen_helper_neon_widen_u8,
2695	gen_helper_neon_widen_u16,
2696	tcg_gen_extu_i32_i64,
2697	NULL,
2698	};
2699	NeonGenWidenFn *widenfn = widenfns[a->size];
2700
2701	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2702	return false;
2703	}
2704
2705	/* UNDEF accesses to D16-D31 if they don't exist. */
2706	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
2707	return false;
2708	}
2709
2710	if (a->vd & 1) {
2711	return false;
2712	}
2713
2714	if (!widenfn) {
2715	return false;
2716	}
2717
2718	if (!neon_access_check(s)) {
2719	return true;
2720	}
2721
2722	rd = tcg_temp_new_i64();
2723	rm0 = tcg_temp_new_i32();
2724	rm1 = tcg_temp_new_i32();
2725
2726	read_neon_element32(rm0, a->vm, 0, MO_32);
2727	read_neon_element32(rm1, a->vm, 1, MO_32);
2728
2729	widenfn(rd, rm0);
2730	tcg_gen_shli_i64(rd, rd, 8 << a->size);
2731	write_neon_element64(rd, a->vd, 0, MO_64);
2732	widenfn(rd, rm1);
2733	tcg_gen_shli_i64(rd, rd, 8 << a->size);
2734	write_neon_element64(rd, a->vd, 1, MO_64);
2735	return true;
2736	}
2737
2738	static bool trans_VCVT_B16_F32(DisasContext s, arg_2misc a)
2739	{
2740	TCGv_ptr fpst;
2741	TCGv_i64 tmp;
2742	TCGv_i32 dst0, dst1;
2743
2744	if (!dc_isar_feature(aa32_bf16, s)) {
2745	return false;
2746	}
2747
2748	/* UNDEF accesses to D16-D31 if they don't exist. */
2749	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
2750	return false;
2751	}
2752
2753	if ((a->vm & 1) \|\| (a->size != 1)) {
2754	return false;
2755	}
2756
2757	if (!neon_access_check(s)) {
2758	return true;
2759	}
2760
2761	fpst = fpstatus_ptr(FPST_STD);
2762	tmp = tcg_temp_new_i64();
2763	dst0 = tcg_temp_new_i32();
2764	dst1 = tcg_temp_new_i32();
2765
2766	read_neon_element64(tmp, a->vm, 0, MO_64);
2767	gen_helper_bfcvt_pair(dst0, tmp, fpst);
2768
2769	read_neon_element64(tmp, a->vm, 1, MO_64);
2770	gen_helper_bfcvt_pair(dst1, tmp, fpst);
2771
2772	write_neon_element32(dst0, a->vd, 0, MO_32);
2773	write_neon_element32(dst1, a->vd, 1, MO_32);
2774	return true;
2775	}
2776
2777	static bool trans_VCVT_F16_F32(DisasContext s, arg_2misc a)
2778	{
2779	TCGv_ptr fpst;
2780	TCGv_i32 ahp, tmp, tmp2, tmp3;
2781
2782	if (!arm_dc_feature(s, ARM_FEATURE_NEON) \|\|
2783	!dc_isar_feature(aa32_fp16_spconv, s)) {
2784	return false;
2785	}
2786
2787	/* UNDEF accesses to D16-D31 if they don't exist. */
2788	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
2789	return false;
2790	}
2791
2792	if ((a->vm & 1) \|\| (a->size != 1)) {
2793	return false;
2794	}
2795
2796	if (!neon_access_check(s)) {
2797	return true;
2798	}
2799
2800	fpst = fpstatus_ptr(FPST_STD);
2801	ahp = get_ahp_flag();
2802	tmp = tcg_temp_new_i32();
2803	read_neon_element32(tmp, a->vm, 0, MO_32);
2804	gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp);
2805	tmp2 = tcg_temp_new_i32();
2806	read_neon_element32(tmp2, a->vm, 1, MO_32);
2807	gen_helper_vfp_fcvt_f32_to_f16(tmp2, tmp2, fpst, ahp);
2808	tcg_gen_shli_i32(tmp2, tmp2, 16);
2809	tcg_gen_or_i32(tmp2, tmp2, tmp);
2810	read_neon_element32(tmp, a->vm, 2, MO_32);
2811	gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp);
2812	tmp3 = tcg_temp_new_i32();
2813	read_neon_element32(tmp3, a->vm, 3, MO_32);
2814	write_neon_element32(tmp2, a->vd, 0, MO_32);
2815	gen_helper_vfp_fcvt_f32_to_f16(tmp3, tmp3, fpst, ahp);
2816	tcg_gen_shli_i32(tmp3, tmp3, 16);
2817	tcg_gen_or_i32(tmp3, tmp3, tmp);
2818	write_neon_element32(tmp3, a->vd, 1, MO_32);
2819	return true;
2820	}
2821
2822	static bool trans_VCVT_F32_F16(DisasContext s, arg_2misc a)
2823	{
2824	TCGv_ptr fpst;
2825	TCGv_i32 ahp, tmp, tmp2, tmp3;
2826
2827	if (!arm_dc_feature(s, ARM_FEATURE_NEON) \|\|
2828	!dc_isar_feature(aa32_fp16_spconv, s)) {
2829	return false;
2830	}
2831
2832	/* UNDEF accesses to D16-D31 if they don't exist. */
2833	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
2834	return false;
2835	}
2836
2837	if ((a->vd & 1) \|\| (a->size != 1)) {
2838	return false;
2839	}
2840
2841	if (!neon_access_check(s)) {
2842	return true;
2843	}
2844
2845	fpst = fpstatus_ptr(FPST_STD);
2846	ahp = get_ahp_flag();
2847	tmp3 = tcg_temp_new_i32();
2848	tmp2 = tcg_temp_new_i32();
2849	tmp = tcg_temp_new_i32();
2850	read_neon_element32(tmp, a->vm, 0, MO_32);
2851	read_neon_element32(tmp2, a->vm, 1, MO_32);
2852	tcg_gen_ext16u_i32(tmp3, tmp);
2853	gen_helper_vfp_fcvt_f16_to_f32(tmp3, tmp3, fpst, ahp);
2854	write_neon_element32(tmp3, a->vd, 0, MO_32);
2855	tcg_gen_shri_i32(tmp, tmp, 16);
2856	gen_helper_vfp_fcvt_f16_to_f32(tmp, tmp, fpst, ahp);
2857	write_neon_element32(tmp, a->vd, 1, MO_32);
2858	tcg_gen_ext16u_i32(tmp3, tmp2);
2859	gen_helper_vfp_fcvt_f16_to_f32(tmp3, tmp3, fpst, ahp);
2860	write_neon_element32(tmp3, a->vd, 2, MO_32);
2861	tcg_gen_shri_i32(tmp2, tmp2, 16);
2862	gen_helper_vfp_fcvt_f16_to_f32(tmp2, tmp2, fpst, ahp);
2863	write_neon_element32(tmp2, a->vd, 3, MO_32);
2864	return true;
2865	}
2866
2867	static bool do_2misc_vec(DisasContext s, arg_2misc a, GVecGen2Fn *fn)
2868	{
2869	int vec_size = a->q ? 16 : 8;
2870	int rd_ofs = neon_full_reg_offset(a->vd);
2871	int rm_ofs = neon_full_reg_offset(a->vm);
2872
2873	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
2874	return false;
2875	}
2876
2877	/* UNDEF accesses to D16-D31 if they don't exist. */
2878	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
2879	return false;
2880	}
2881
2882	if (a->size == 3) {
2883	return false;
2884	}
2885
2886	if ((a->vd \| a->vm) & a->q) {
2887	return false;
2888	}
2889
2890	if (!neon_access_check(s)) {
2891	return true;
2892	}
2893
2894	fn(a->size, rd_ofs, rm_ofs, vec_size, vec_size);
2895
2896	return true;
2897	}
2898
2899	#define DO_2MISC_VEC(INSN, FN) \
2900	static bool trans_##INSN(DisasContext s, arg_2misc a) \
2901	{ \
2902	return do_2misc_vec(s, a, FN); \
2903	}
2904
2905	DO_2MISC_VEC(VNEG, tcg_gen_gvec_neg)
2906	DO_2MISC_VEC(VABS, tcg_gen_gvec_abs)
2907	DO_2MISC_VEC(VCEQ0, gen_gvec_ceq0)
2908	DO_2MISC_VEC(VCGT0, gen_gvec_cgt0)
2909	DO_2MISC_VEC(VCLE0, gen_gvec_cle0)
2910	DO_2MISC_VEC(VCGE0, gen_gvec_cge0)
2911	DO_2MISC_VEC(VCLT0, gen_gvec_clt0)
2912	DO_2MISC_VEC(VCLS, gen_gvec_cls)
2913	DO_2MISC_VEC(VCLZ, gen_gvec_clz)
2914	DO_2MISC_VEC(VREV64, gen_gvec_rev64)
2915	DO_2MISC_VEC(VPADDL_S, gen_gvec_saddlp)
2916	DO_2MISC_VEC(VPADDL_U, gen_gvec_uaddlp)
2917	DO_2MISC_VEC(VPADAL_S, gen_gvec_sadalp)
2918	DO_2MISC_VEC(VPADAL_U, gen_gvec_uadalp)
2919
2920	static bool trans_VMVN(DisasContext s, arg_2misc a)
2921	{
2922	if (a->size != 0) {
2923	return false;
2924	}
2925	return do_2misc_vec(s, a, tcg_gen_gvec_not);
2926	}
2927
2928	static bool trans_VCNT(DisasContext s, arg_2misc a)
2929	{
2930	if (a->size != 0) {
2931	return false;
2932	}
2933	return do_2misc_vec(s, a, gen_gvec_cnt);
2934	}
2935
2936	static bool trans_VREV16(DisasContext s, arg_2misc a)
2937	{
2938	if (a->size != 0) {
2939	return false;
2940	}
2941	return do_2misc_vec(s, a, gen_gvec_rev16);
2942	}
2943
2944	static bool trans_VREV32(DisasContext s, arg_2misc a)
2945	{
2946	if (a->size != 0 && a->size != 1) {
2947	return false;
2948	}
2949	return do_2misc_vec(s, a, gen_gvec_rev32);
2950	}
2951
2952	#define WRAP_2M_3_OOL_FN(WRAPNAME, FUNC, DATA) \
2953	static void WRAPNAME(unsigned vece, uint32_t rd_ofs, \
2954	uint32_t rm_ofs, uint32_t oprsz, \
2955	uint32_t maxsz) \
2956	{ \
2957	tcg_gen_gvec_3_ool(rd_ofs, rd_ofs, rm_ofs, oprsz, maxsz, \
2958	DATA, FUNC); \
2959	}
2960
2961	#define WRAP_2M_2_OOL_FN(WRAPNAME, FUNC, DATA) \
2962	static void WRAPNAME(unsigned vece, uint32_t rd_ofs, \
2963	uint32_t rm_ofs, uint32_t oprsz, \
2964	uint32_t maxsz) \
2965	{ \
2966	tcg_gen_gvec_2_ool(rd_ofs, rm_ofs, oprsz, maxsz, DATA, FUNC); \
2967	}
2968
2969	WRAP_2M_3_OOL_FN(gen_AESE, gen_helper_crypto_aese, 0)
2970	WRAP_2M_3_OOL_FN(gen_AESD, gen_helper_crypto_aesd, 0)
2971	WRAP_2M_2_OOL_FN(gen_AESMC, gen_helper_crypto_aesmc, 0)
2972	WRAP_2M_2_OOL_FN(gen_AESIMC, gen_helper_crypto_aesimc, 0)
2973	WRAP_2M_2_OOL_FN(gen_SHA1H, gen_helper_crypto_sha1h, 0)
2974	WRAP_2M_2_OOL_FN(gen_SHA1SU1, gen_helper_crypto_sha1su1, 0)
2975	WRAP_2M_2_OOL_FN(gen_SHA256SU0, gen_helper_crypto_sha256su0, 0)
2976
2977	#define DO_2M_CRYPTO(INSN, FEATURE, SIZE) \
2978	static bool trans_##INSN(DisasContext s, arg_2misc a) \
2979	{ \
2980	if (!dc_isar_feature(FEATURE, s) \|\| a->size != SIZE) { \
2981	return false; \
2982	} \
2983	return do_2misc_vec(s, a, gen_##INSN); \
2984	}
2985
2986	DO_2M_CRYPTO(AESE, aa32_aes, 0)
2987	DO_2M_CRYPTO(AESD, aa32_aes, 0)
2988	DO_2M_CRYPTO(AESMC, aa32_aes, 0)
2989	DO_2M_CRYPTO(AESIMC, aa32_aes, 0)
2990	DO_2M_CRYPTO(SHA1H, aa32_sha1, 2)
2991	DO_2M_CRYPTO(SHA1SU1, aa32_sha1, 2)
2992	DO_2M_CRYPTO(SHA256SU0, aa32_sha2, 2)
2993
2994	static bool do_2misc(DisasContext s, arg_2misc a, NeonGenOneOpFn *fn)
2995	{
2996	TCGv_i32 tmp;
2997	int pass;
2998
2999	/* Handle a 2-reg-misc operation by iterating 32 bits at a time */
3000	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
3001	return false;
3002	}
3003
3004	/* UNDEF accesses to D16-D31 if they don't exist. */
3005	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
3006	return false;
3007	}
3008
3009	if (!fn) {
3010	return false;
3011	}
3012
3013	if ((a->vd \| a->vm) & a->q) {
3014	return false;
3015	}
3016
3017	if (!neon_access_check(s)) {
3018	return true;
3019	}
3020
3021	tmp = tcg_temp_new_i32();
3022	for (pass = 0; pass < (a->q ? 4 : 2); pass++) {
3023	read_neon_element32(tmp, a->vm, pass, MO_32);
3024	fn(tmp, tmp);
3025	write_neon_element32(tmp, a->vd, pass, MO_32);
3026	}
3027	return true;
3028	}
3029
3030	static bool trans_VABS_F(DisasContext s, arg_2misc a)
3031	{
3032	if (a->size == MO_16) {
3033	if (!dc_isar_feature(aa32_fp16_arith, s)) {
3034	return false;
3035	}
3036	} else if (a->size != MO_32) {
3037	return false;
3038	}
3039	return do_2misc_vec(s, a, gen_gvec_fabs);
3040	}
3041
3042	static bool trans_VNEG_F(DisasContext s, arg_2misc a)
3043	{
3044	if (a->size == MO_16) {
3045	if (!dc_isar_feature(aa32_fp16_arith, s)) {
3046	return false;
3047	}
3048	} else if (a->size != MO_32) {
3049	return false;
3050	}
3051	return do_2misc_vec(s, a, gen_gvec_fneg);
3052	}
3053
3054	static bool trans_VRECPE(DisasContext s, arg_2misc a)
3055	{
3056	if (a->size != 2) {
3057	return false;
3058	}
3059	return do_2misc_vec(s, a, gen_gvec_urecpe);
3060	}
3061
3062	static bool trans_VRSQRTE(DisasContext s, arg_2misc a)
3063	{
3064	if (a->size != 2) {
3065	return false;
3066	}
3067	return do_2misc_vec(s, a, gen_gvec_ursqrte);
3068	}
3069
3070	#define WRAP_1OP_ENV_FN(WRAPNAME, FUNC) \
3071	static void WRAPNAME(TCGv_i32 d, TCGv_i32 m) \
3072	{ \
3073	FUNC(d, tcg_env, m); \
3074	}
3075
3076	WRAP_1OP_ENV_FN(gen_VQABS_s8, gen_helper_neon_qabs_s8)
3077	WRAP_1OP_ENV_FN(gen_VQABS_s16, gen_helper_neon_qabs_s16)
3078	WRAP_1OP_ENV_FN(gen_VQABS_s32, gen_helper_neon_qabs_s32)
3079	WRAP_1OP_ENV_FN(gen_VQNEG_s8, gen_helper_neon_qneg_s8)
3080	WRAP_1OP_ENV_FN(gen_VQNEG_s16, gen_helper_neon_qneg_s16)
3081	WRAP_1OP_ENV_FN(gen_VQNEG_s32, gen_helper_neon_qneg_s32)
3082
3083	static bool trans_VQABS(DisasContext s, arg_2misc a)
3084	{
3085	static NeonGenOneOpFn * const fn[] = {
3086	gen_VQABS_s8,
3087	gen_VQABS_s16,
3088	gen_VQABS_s32,
3089	NULL,
3090	};
3091	return do_2misc(s, a, fn[a->size]);
3092	}
3093
3094	static bool trans_VQNEG(DisasContext s, arg_2misc a)
3095	{
3096	static NeonGenOneOpFn * const fn[] = {
3097	gen_VQNEG_s8,
3098	gen_VQNEG_s16,
3099	gen_VQNEG_s32,
3100	NULL,
3101	};
3102	return do_2misc(s, a, fn[a->size]);
3103	}
3104
3105	#define DO_2MISC_FP_VEC(INSN, HFUNC, SFUNC) \
3106	static void gen_##INSN(unsigned vece, uint32_t rd_ofs, \
3107	uint32_t rm_ofs, \
3108	uint32_t oprsz, uint32_t maxsz) \
3109	{ \
3110	static gen_helper_gvec_2_ptr * const fns[4] = { \
3111	NULL, HFUNC, SFUNC, NULL, \
3112	}; \
3113	TCGv_ptr fpst; \
3114	fpst = fpstatus_ptr(vece == MO_16 ? FPST_STD_F16 : FPST_STD); \
3115	tcg_gen_gvec_2_ptr(rd_ofs, rm_ofs, fpst, oprsz, maxsz, 0, \
3116	fns[vece]); \
3117	} \
3118	static bool trans_##INSN(DisasContext s, arg_2misc a) \
3119	{ \
3120	if (a->size == MO_16) { \
3121	if (!dc_isar_feature(aa32_fp16_arith, s)) { \
3122	return false; \
3123	} \
3124	} else if (a->size != MO_32) { \
3125	return false; \
3126	} \
3127	return do_2misc_vec(s, a, gen_##INSN); \
3128	}
3129
3130	DO_2MISC_FP_VEC(VRECPE_F, gen_helper_gvec_frecpe_h, gen_helper_gvec_frecpe_s)
3131	DO_2MISC_FP_VEC(VRSQRTE_F, gen_helper_gvec_frsqrte_h, gen_helper_gvec_frsqrte_s)
3132	DO_2MISC_FP_VEC(VCGT0_F, gen_helper_gvec_fcgt0_h, gen_helper_gvec_fcgt0_s)
3133	DO_2MISC_FP_VEC(VCGE0_F, gen_helper_gvec_fcge0_h, gen_helper_gvec_fcge0_s)
3134	DO_2MISC_FP_VEC(VCEQ0_F, gen_helper_gvec_fceq0_h, gen_helper_gvec_fceq0_s)
3135	DO_2MISC_FP_VEC(VCLT0_F, gen_helper_gvec_fclt0_h, gen_helper_gvec_fclt0_s)
3136	DO_2MISC_FP_VEC(VCLE0_F, gen_helper_gvec_fcle0_h, gen_helper_gvec_fcle0_s)
3137	DO_2MISC_FP_VEC(VCVT_FS, gen_helper_gvec_sstoh, gen_helper_gvec_sitos)
3138	DO_2MISC_FP_VEC(VCVT_FU, gen_helper_gvec_ustoh, gen_helper_gvec_uitos)
3139	DO_2MISC_FP_VEC(VCVT_SF, gen_helper_gvec_tosszh, gen_helper_gvec_tosizs)
3140	DO_2MISC_FP_VEC(VCVT_UF, gen_helper_gvec_touszh, gen_helper_gvec_touizs)
3141
3142	DO_2MISC_FP_VEC(VRINTX_impl, gen_helper_gvec_vrintx_h, gen_helper_gvec_vrintx_s)
3143
3144	static bool trans_VRINTX(DisasContext s, arg_2misc a)
3145	{
3146	if (!arm_dc_feature(s, ARM_FEATURE_V8)) {
3147	return false;
3148	}
3149	return trans_VRINTX_impl(s, a);
3150	}
3151
3152	#define DO_VEC_RMODE(INSN, RMODE, OP) \
3153	static void gen_##INSN(unsigned vece, uint32_t rd_ofs, \
3154	uint32_t rm_ofs, \
3155	uint32_t oprsz, uint32_t maxsz) \
3156	{ \
3157	static gen_helper_gvec_2_ptr * const fns[4] = { \
3158	NULL, \
3159	gen_helper_gvec_##OP##h, \
3160	gen_helper_gvec_##OP##s, \
3161	NULL, \
3162	}; \
3163	TCGv_ptr fpst; \
3164	fpst = fpstatus_ptr(vece == 1 ? FPST_STD_F16 : FPST_STD); \
3165	tcg_gen_gvec_2_ptr(rd_ofs, rm_ofs, fpst, oprsz, maxsz, \
3166	arm_rmode_to_sf(RMODE), fns[vece]); \
3167	} \
3168	static bool trans_##INSN(DisasContext s, arg_2misc a) \
3169	{ \
3170	if (!arm_dc_feature(s, ARM_FEATURE_V8)) { \
3171	return false; \
3172	} \
3173	if (a->size == MO_16) { \
3174	if (!dc_isar_feature(aa32_fp16_arith, s)) { \
3175	return false; \
3176	} \
3177	} else if (a->size != MO_32) { \
3178	return false; \
3179	} \
3180	return do_2misc_vec(s, a, gen_##INSN); \
3181	}
3182
3183	DO_VEC_RMODE(VCVTAU, FPROUNDING_TIEAWAY, vcvt_rm_u)
3184	DO_VEC_RMODE(VCVTAS, FPROUNDING_TIEAWAY, vcvt_rm_s)
3185	DO_VEC_RMODE(VCVTNU, FPROUNDING_TIEEVEN, vcvt_rm_u)
3186	DO_VEC_RMODE(VCVTNS, FPROUNDING_TIEEVEN, vcvt_rm_s)
3187	DO_VEC_RMODE(VCVTPU, FPROUNDING_POSINF, vcvt_rm_u)
3188	DO_VEC_RMODE(VCVTPS, FPROUNDING_POSINF, vcvt_rm_s)
3189	DO_VEC_RMODE(VCVTMU, FPROUNDING_NEGINF, vcvt_rm_u)
3190	DO_VEC_RMODE(VCVTMS, FPROUNDING_NEGINF, vcvt_rm_s)
3191
3192	DO_VEC_RMODE(VRINTN, FPROUNDING_TIEEVEN, vrint_rm_)
3193	DO_VEC_RMODE(VRINTA, FPROUNDING_TIEAWAY, vrint_rm_)
3194	DO_VEC_RMODE(VRINTZ, FPROUNDING_ZERO, vrint_rm_)
3195	DO_VEC_RMODE(VRINTM, FPROUNDING_NEGINF, vrint_rm_)
3196	DO_VEC_RMODE(VRINTP, FPROUNDING_POSINF, vrint_rm_)
3197
3198	static bool trans_VSWP(DisasContext s, arg_2misc a)
3199	{
3200	TCGv_i64 rm, rd;
3201	int pass;
3202
3203	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
3204	return false;
3205	}
3206
3207	/* UNDEF accesses to D16-D31 if they don't exist. */
3208	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
3209	return false;
3210	}
3211
3212	if (a->size != 0) {
3213	return false;
3214	}
3215
3216	if ((a->vd \| a->vm) & a->q) {
3217	return false;
3218	}
3219
3220	if (!neon_access_check(s)) {
3221	return true;
3222	}
3223
3224	rm = tcg_temp_new_i64();
3225	rd = tcg_temp_new_i64();
3226	for (pass = 0; pass < (a->q ? 2 : 1); pass++) {
3227	read_neon_element64(rm, a->vm, pass, MO_64);
3228	read_neon_element64(rd, a->vd, pass, MO_64);
3229	write_neon_element64(rm, a->vd, pass, MO_64);
3230	write_neon_element64(rd, a->vm, pass, MO_64);
3231	}
3232	return true;
3233	}
3234
3235	static void gen_neon_trn_u8(TCGv_i32 t0, TCGv_i32 t1)
3236	{
3237	TCGv_i32 rd, tmp;
3238
3239	rd = tcg_temp_new_i32();
3240	tmp = tcg_temp_new_i32();
3241
3242	tcg_gen_shli_i32(rd, t0, 8);
3243	tcg_gen_andi_i32(rd, rd, 0xff00ff00);
3244	tcg_gen_andi_i32(tmp, t1, 0x00ff00ff);
3245	tcg_gen_or_i32(rd, rd, tmp);
3246
3247	tcg_gen_shri_i32(t1, t1, 8);
3248	tcg_gen_andi_i32(t1, t1, 0x00ff00ff);
3249	tcg_gen_andi_i32(tmp, t0, 0xff00ff00);
3250	tcg_gen_or_i32(t1, t1, tmp);
3251	tcg_gen_mov_i32(t0, rd);
3252	}
3253
3254	static void gen_neon_trn_u16(TCGv_i32 t0, TCGv_i32 t1)
3255	{
3256	TCGv_i32 rd, tmp;
3257
3258	rd = tcg_temp_new_i32();
3259	tmp = tcg_temp_new_i32();
3260
3261	tcg_gen_shli_i32(rd, t0, 16);
3262	tcg_gen_andi_i32(tmp, t1, 0xffff);
3263	tcg_gen_or_i32(rd, rd, tmp);
3264	tcg_gen_shri_i32(t1, t1, 16);
3265	tcg_gen_andi_i32(tmp, t0, 0xffff0000);
3266	tcg_gen_or_i32(t1, t1, tmp);
3267	tcg_gen_mov_i32(t0, rd);
3268	}
3269
3270	static bool trans_VTRN(DisasContext s, arg_2misc a)
3271	{
3272	TCGv_i32 tmp, tmp2;
3273	int pass;
3274
3275	if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
3276	return false;
3277	}
3278
3279	/* UNDEF accesses to D16-D31 if they don't exist. */
3280	if (!neon_dregs_ok(s, a->vd \| a->vm)) {
3281	return false;
3282	}
3283
3284	if ((a->vd \| a->vm) & a->q) {
3285	return false;
3286	}
3287
3288	if (a->size == 3) {
3289	return false;
3290	}
3291
3292	if (!neon_access_check(s)) {
3293	return true;
3294	}
3295
3296	tmp = tcg_temp_new_i32();
3297	tmp2 = tcg_temp_new_i32();
3298	if (a->size == MO_32) {
3299	for (pass = 0; pass < (a->q ? 4 : 2); pass += 2) {
3300	read_neon_element32(tmp, a->vm, pass, MO_32);
3301	read_neon_element32(tmp2, a->vd, pass + 1, MO_32);
3302	write_neon_element32(tmp2, a->vm, pass, MO_32);
3303	write_neon_element32(tmp, a->vd, pass + 1, MO_32);
3304	}
3305	} else {
3306	for (pass = 0; pass < (a->q ? 4 : 2); pass++) {
3307	read_neon_element32(tmp, a->vm, pass, MO_32);
3308	read_neon_element32(tmp2, a->vd, pass, MO_32);
3309	if (a->size == MO_8) {
3310	gen_neon_trn_u8(tmp, tmp2);
3311	} else {
3312	gen_neon_trn_u16(tmp, tmp2);
3313	}
3314	write_neon_element32(tmp2, a->vm, pass, MO_32);
3315	write_neon_element32(tmp, a->vd, pass, MO_32);
3316	}
3317	}
3318	return true;
3319	}
3320
3321	static bool trans_VSMMLA(DisasContext s, arg_VSMMLA a)
3322	{
3323	if (!dc_isar_feature(aa32_i8mm, s)) {
3324	return false;
3325	}
3326	return do_neon_ddda(s, 7, a->vd, a->vn, a->vm, 0,
3327	gen_helper_gvec_smmla_b);
3328	}
3329
3330	static bool trans_VUMMLA(DisasContext s, arg_VUMMLA a)
3331	{
3332	if (!dc_isar_feature(aa32_i8mm, s)) {
3333	return false;
3334	}
3335	return do_neon_ddda(s, 7, a->vd, a->vn, a->vm, 0,
3336	gen_helper_gvec_ummla_b);
3337	}
3338
3339	static bool trans_VUSMMLA(DisasContext s, arg_VUSMMLA a)
3340	{
3341	if (!dc_isar_feature(aa32_i8mm, s)) {
3342	return false;
3343	}
3344	return do_neon_ddda(s, 7, a->vd, a->vn, a->vm, 0,
3345	gen_helper_gvec_usmmla_b);
3346	}
3347
3348	static bool trans_VMMLA_b16(DisasContext s, arg_VMMLA_b16 a)
3349	{
3350	if (!dc_isar_feature(aa32_bf16, s)) {
3351	return false;
3352	}
3353	return do_neon_ddda_env(s, 7, a->vd, a->vn, a->vm, 0,
3354	gen_helper_gvec_bfmmla);
3355	}
3356
3357	static bool trans_VFMA_b16(DisasContext s, arg_VFMA_b16 a)
3358	{
3359	if (!dc_isar_feature(aa32_bf16, s)) {
3360	return false;
3361	}
3362	return do_neon_ddda_fpst(s, 7, a->vd, a->vn, a->vm, a->q, FPST_STD,
3363	gen_helper_gvec_bfmlal);
3364	}
3365
3366	static bool trans_VFMA_b16_scal(DisasContext s, arg_VFMA_b16_scal a)
3367	{
3368	if (!dc_isar_feature(aa32_bf16, s)) {
3369	return false;
3370	}
3371	return do_neon_ddda_fpst(s, 6, a->vd, a->vn, a->vm,
3372	(a->index << 1) \| a->q, FPST_STD,
3373	gen_helper_gvec_bfmlal_idx);
3374	}