translate-sme.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 2,231 lines 80.4 KB

Raw

1	/*
2	* AArch64 SME translation
3	*
4	* Copyright (c) 2022 Linaro, Ltd
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#include "qemu/osdep.h"
21	#include "cpu.h"
22	#include "helper-a64.h"
23	#include "helper-sme.h"
24	#include "helper-sve.h"
25	#include "helper-fp8.h"
26	#include "translate.h"
27	#include "translate-a64.h"
28	#include "tcg/tcg-op.h"
29
30	/*
31	* Include the generated decoder.
32	*/
33
34	static int expand_tmop_zk(DisasContext *s, int x)
35	{
36	/* Pseudocode for 1:K:1:zk. */
37	return 0b10100 \| ((x & 4) << 1) \| (x & 3);
38	}
39
40	#include "decode-sme.c.inc"
41
42	static bool sme2_zt0_enabled_check(DisasContext *s)
43	{
44	if (!sme_za_enabled_check(s)) {
45	return false;
46	}
47	if (s->zt0_excp_el) {
48	gen_exception_insn_el(s, 0, EXCP_UDEF,
49	syn_smetrap(SME_ET_InaccessibleZT0, false),
50	s->zt0_excp_el);
51	return false;
52	}
53	return true;
54	}
55
56	/* Resolve tile.size[rs+imm] to a host pointer. */
57	static TCGv_ptr get_tile_rowcol(DisasContext *s, int esz, int rs,
58	int tile, int imm, int div_len,
59	int vec_mod, bool vertical)
60	{
61	int pos, len, offset;
62	TCGv_i32 tmp;
63	TCGv_ptr addr;
64
65	/* Compute the final index, which is Rs+imm. */
66	tmp = tcg_temp_new_i32();
67	tcg_gen_trunc_tl_i32(tmp, cpu_reg(s, rs));
68	/*
69	* Round the vector index down to a multiple of vec_mod if necessary.
70	* We do this before adding the offset, to handle cases like
71	* MOVA (tile to vector, 2 registers) where we want to call this
72	* several times in a loop with an increasing offset. We rely on
73	* the instruction encodings always forcing the initial offset in
74	* [rs + offset] to be a multiple of vec_mod. The pseudocode usually
75	* does the round-down after adding the offset rather than before,
76	* but MOVA is an exception.
77	*/
78	if (vec_mod > 1) {
79	tcg_gen_andc_i32(tmp, tmp, tcg_constant_i32(vec_mod - 1));
80	}
81	tcg_gen_addi_i32(tmp, tmp, imm);
82
83	/* Prepare a power-of-two modulo via extraction of @len bits. */
84	len = ctz32(streaming_vec_reg_size(s) / div_len) - esz;
85
86	if (!len) {
87	/*
88	* SVL is 128 and the element size is 128. There is exactly
89	* one 128x128 tile in the ZA storage, and so we calculate
90	* (Rs + imm) MOD 1, which is always 0. We need to special case
91	* this because TCG doesn't allow deposit ops with len 0.
92	*/
93	tcg_gen_movi_i32(tmp, 0);
94	} else if (vertical) {
95	/*
96	* Compute the byte offset of the index within the tile:
97	* (index % (svl / size)) * size
98	* = (index % (svl >> esz)) << esz
99	* Perform the power-of-two modulo via extraction of the low @len bits.
100	* Perform the multiply by shifting left by @pos bits.
101	* Perform these operations simultaneously via deposit into zero.
102	*/
103	pos = esz;
104	tcg_gen_deposit_z_i32(tmp, tmp, pos, len);
105
106	/*
107	* For big-endian, adjust the indexed column byte offset within
108	* the uint64_t host words that make up env->zarray[].
109	*/
110	if (HOST_BIG_ENDIAN && esz < MO_64) {
111	tcg_gen_xori_i32(tmp, tmp, 8 - (1 << esz));
112	}
113	} else {
114	/*
115	* Compute the byte offset of the index within the tile:
116	* (index % (svl / size)) * (size * sizeof(row))
117	* = (index % (svl >> esz)) << (esz + log2(sizeof(row)))
118	*/
119	pos = esz + ctz32(sizeof(ARMVectorReg));
120	tcg_gen_deposit_z_i32(tmp, tmp, pos, len);
121
122	/* Row slices are always aligned and need no endian adjustment. */
123	}
124
125	/* The tile byte offset within env->zarray is the row. */
126	offset = tile * sizeof(ARMVectorReg);
127
128	/* Include the byte offset of zarray to make this relative to env. */
129	offset += offsetof(CPUARMState, za_state.za);
130	tcg_gen_addi_i32(tmp, tmp, offset);
131
132	/* Add the byte offset to env to produce the final pointer. */
133	addr = tcg_temp_new_ptr();
134	tcg_gen_ext_i32_ptr(addr, tmp);
135	tcg_gen_add_ptr(addr, addr, tcg_env);
136
137	return addr;
138	}
139
140	/* Resolve ZArray[rs+imm] to a host pointer. */
141	static TCGv_ptr get_zarray(DisasContext *s, int rs, int imm,
142	int div_len, int vec_mod)
143	{
144	/* ZA[n] equates to ZA0H.B[n]. */
145	return get_tile_rowcol(s, MO_8, rs, 0, imm, div_len, vec_mod, false);
146	}
147
148	/*
149	* Resolve tile.size[0] to a host pointer.
150	* Used by e.g. outer product insns where we require the entire tile.
151	*/
152	static TCGv_ptr get_tile(DisasContext *s, int esz, int tile)
153	{
154	TCGv_ptr addr = tcg_temp_new_ptr();
155	int offset;
156
157	offset = tile * sizeof(ARMVectorReg) + offsetof(CPUARMState, za_state.za);
158
159	tcg_gen_addi_ptr(addr, tcg_env, offset);
160	return addr;
161	}
162
163	static bool trans_ZERO(DisasContext s, arg_ZERO a)
164	{
165	if (!dc_isar_feature(aa64_sme, s)) {
166	return false;
167	}
168	if (sme_za_enabled_check(s)) {
169	gen_helper_sme_zero(tcg_env, tcg_constant_i32(a->imm),
170	tcg_constant_i32(streaming_vec_reg_size(s)));
171	}
172	return true;
173	}
174
175	static bool trans_ZERO_zt0(DisasContext s, arg_ZERO_zt0 a)
176	{
177	if (!dc_isar_feature(aa64_sme2, s)) {
178	return false;
179	}
180	if (sme_enabled_check(s) && sme2_zt0_enabled_check(s)) {
181	tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUARMState, za_state.zt0),
182	sizeof_field(CPUARMState, za_state.zt0),
183	sizeof_field(CPUARMState, za_state.zt0), 0);
184	}
185	return true;
186	}
187
188	static bool trans_ZERO_za(DisasContext s, arg_ZERO_za a)
189	{
190	if (!dc_isar_feature(aa64_sme2p1, s)) {
191	return false;
192	}
193	if (sme_smza_enabled_check(s)) {
194	int svl = streaming_vec_reg_size(s);
195	int vstride = svl / a->ngrp;
196	TCGv_ptr t_za = get_zarray(s, a->rv, a->off, a->ngrp, a->nvec);
197
198	for (int r = 0; r < a->ngrp; ++r) {
199	for (int i = 0; i < a->nvec; ++i) {
200	int o_za = (r * vstride + i) * sizeof(ARMVectorReg);
201	tcg_gen_gvec_dup_imm_var(MO_64, t_za, o_za, svl, svl, 0);
202	}
203	}
204	}
205	return true;
206	}
207
208	static bool do_mova_tile(DisasContext s, arg_mova_p a, bool to_vec)
209	{
210	static gen_helper_gvec_4 * const h_fns[5] = {
211	gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
212	gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d,
213	gen_helper_sve_sel_zpzz_q
214	};
215	static gen_helper_gvec_3 * const cz_fns[5] = {
216	gen_helper_sme_mova_cz_b, gen_helper_sme_mova_cz_h,
217	gen_helper_sme_mova_cz_s, gen_helper_sme_mova_cz_d,
218	gen_helper_sme_mova_cz_q,
219	};
220	static gen_helper_gvec_3 * const zc_fns[5] = {
221	gen_helper_sme_mova_zc_b, gen_helper_sme_mova_zc_h,
222	gen_helper_sme_mova_zc_s, gen_helper_sme_mova_zc_d,
223	gen_helper_sme_mova_zc_q,
224	};
225
226	TCGv_ptr t_za, t_zr, t_pg;
227	TCGv_i32 t_desc;
228	int svl;
229
230	if (!sme_smza_enabled_check(s)) {
231	return true;
232	}
233
234	t_za = get_tile_rowcol(s, a->esz, a->rs, a->za, a->off, 1, 0, a->v);
235	t_zr = vec_full_reg_ptr(s, a->zr);
236	t_pg = pred_full_reg_ptr(s, a->pg);
237
238	svl = streaming_vec_reg_size(s);
239	t_desc = tcg_constant_i32(simd_desc(svl, svl, 0));
240
241	if (a->v) {
242	/* Vertical slice -- use sme mova helpers. */
243	if (to_vec) {
244	zc_fns[a->esz](t_zr, t_za, t_pg, t_desc);
245	} else {
246	cz_fns[a->esz](t_za, t_zr, t_pg, t_desc);
247	}
248	} else {
249	/* Horizontal slice -- reuse sve sel helpers. */
250	if (to_vec) {
251	h_fns[a->esz](t_zr, t_za, t_zr, t_pg, t_desc);
252	} else {
253	h_fns[a->esz](t_za, t_zr, t_za, t_pg, t_desc);
254	}
255	}
256	return true;
257	}
258
259	TRANS_FEAT(MOVA_tz, aa64_sme, do_mova_tile, a, false)
260	TRANS_FEAT(MOVA_zt, aa64_sme, do_mova_tile, a, true)
261
262	static bool do_mova_tile_n(DisasContext s, arg_mova_t a, int n,
263	bool to_vec, bool zero)
264	{
265	static gen_helper_gvec_2 * const cz_fns[] = {
266	gen_helper_sme2_mova_cz_b, gen_helper_sme2_mova_cz_h,
267	gen_helper_sme2_mova_cz_s, gen_helper_sme2_mova_cz_d,
268	};
269	static gen_helper_gvec_2 * const zc_fns[] = {
270	gen_helper_sme2_mova_zc_b, gen_helper_sme2_mova_zc_h,
271	gen_helper_sme2_mova_zc_s, gen_helper_sme2_mova_zc_d,
272	};
273	static gen_helper_gvec_2 * const zc_z_fns[] = {
274	gen_helper_sme2p1_movaz_zc_b, gen_helper_sme2p1_movaz_zc_h,
275	gen_helper_sme2p1_movaz_zc_s, gen_helper_sme2p1_movaz_zc_d,
276	gen_helper_sme2p1_movaz_zc_q,
277	};
278	TCGv_ptr t_za;
279	int svl, bytes_per_op = n << a->esz;
280
281	/*
282	* The MaxImplementedSVL check happens in the decode pseudocode,
283	* before the SM+ZA enabled check in the operation pseudocode.
284	* This will (currently) only fail for NREG=4, ESZ=MO_64.
285	*/
286	if (s->max_svl < bytes_per_op) {
287	unallocated_encoding(s);
288	return true;
289	}
290
291	assert(a->esz <= MO_64 + zero);
292
293	if (!sme_smza_enabled_check(s)) {
294	return true;
295	}
296
297	svl = streaming_vec_reg_size(s);
298
299	/*
300	* The CurrentVL check happens in the operation pseudocode,
301	* after the SM+ZA enabled check.
302	*/
303	if (svl < bytes_per_op) {
304	unallocated_encoding(s);
305	return true;
306	}
307
308	if (a->v) {
309	TCGv_i32 t_desc = tcg_constant_i32(simd_desc(svl, svl, 0));
310
311	for (int i = 0; i < n; ++i) {
312	TCGv_ptr t_zr = vec_full_reg_ptr(s, a->zr * n + i);
313	t_za = get_tile_rowcol(s, a->esz, a->rs, a->za,
314	a->off * n + i, 1, n, a->v);
315	if (zero) {
316	zc_z_fns[a->esz](t_zr, t_za, t_desc);
317	} else if (to_vec) {
318	zc_fns[a->esz](t_zr, t_za, t_desc);
319	} else {
320	cz_fns[a->esz](t_za, t_zr, t_desc);
321	}
322	}
323	} else {
324	for (int i = 0; i < n; ++i) {
325	int o_zr = vec_full_reg_offset(s, a->zr * n + i);
326	t_za = get_tile_rowcol(s, a->esz, a->rs, a->za,
327	a->off * n + i, 1, n, a->v);
328	if (to_vec) {
329	tcg_gen_gvec_mov_var(MO_8, tcg_env, o_zr, t_za, 0, svl, svl);
330	if (zero) {
331	tcg_gen_gvec_dup_imm_var(MO_8, t_za, 0, svl, svl, 0);
332	}
333	} else {
334	tcg_gen_gvec_mov_var(MO_8, t_za, 0, tcg_env, o_zr, svl, svl);
335	}
336	}
337	}
338	return true;
339	}
340
341	TRANS_FEAT(MOVA_tz2, aa64_sme2, do_mova_tile_n, a, 2, false, false)
342	TRANS_FEAT(MOVA_tz4, aa64_sme2, do_mova_tile_n, a, 4, false, false)
343	TRANS_FEAT(MOVA_zt2, aa64_sme2, do_mova_tile_n, a, 2, true, false)
344	TRANS_FEAT(MOVA_zt4, aa64_sme2, do_mova_tile_n, a, 4, true, false)
345
346	TRANS_FEAT(MOVAZ_zt, aa64_sme2p1, do_mova_tile_n, a, 1, true, true)
347	TRANS_FEAT(MOVAZ_zt2, aa64_sme2p1, do_mova_tile_n, a, 2, true, true)
348	TRANS_FEAT(MOVAZ_zt4, aa64_sme2p1, do_mova_tile_n, a, 4, true, true)
349
350	static bool do_mova_array_n(DisasContext s, arg_mova_a a, int n,
351	bool to_vec, bool zero)
352	{
353	TCGv_ptr t_za;
354	int svl;
355
356	if (!sme_smza_enabled_check(s)) {
357	return true;
358	}
359
360	svl = streaming_vec_reg_size(s);
361	t_za = get_zarray(s, a->rv, a->off, n, 0);
362
363	for (int i = 0; i < n; ++i) {
364	int o_za = (svl / n * sizeof(ARMVectorReg)) * i;
365	int o_zr = vec_full_reg_offset(s, a->zr * n + i);
366
367	if (to_vec) {
368	tcg_gen_gvec_mov_var(MO_8, tcg_env, o_zr, t_za, o_za, svl, svl);
369	if (zero) {
370	tcg_gen_gvec_dup_imm_var(MO_8, t_za, o_za, svl, svl, 0);
371	}
372	} else {
373	tcg_gen_gvec_mov_var(MO_8, t_za, o_za, tcg_env, o_zr, svl, svl);
374	}
375	}
376	return true;
377	}
378
379	TRANS_FEAT(MOVA_az2, aa64_sme2, do_mova_array_n, a, 2, false, false)
380	TRANS_FEAT(MOVA_az4, aa64_sme2, do_mova_array_n, a, 4, false, false)
381	TRANS_FEAT(MOVA_za2, aa64_sme2, do_mova_array_n, a, 2, true, false)
382	TRANS_FEAT(MOVA_za4, aa64_sme2, do_mova_array_n, a, 4, true, false)
383
384	TRANS_FEAT(MOVAZ_za2, aa64_sme2p1, do_mova_array_n, a, 2, true, true)
385	TRANS_FEAT(MOVAZ_za4, aa64_sme2p1, do_mova_array_n, a, 4, true, true)
386
387	static bool do_movt(DisasContext s, arg_MOVT_rzt a,
388	void (*func)(TCGv_i64, TCGv_ptr, tcg_target_long))
389	{
390	if (sme2_zt0_enabled_check(s)) {
391	func(cpu_reg(s, a->rt), tcg_env,
392	offsetof(CPUARMState, za_state.zt0) + a->off * 8);
393	}
394	return true;
395	}
396
397	TRANS_FEAT(MOVT_rzt, aa64_sme2, do_movt, a, tcg_gen_ld_i64)
398	TRANS_FEAT(MOVT_ztr, aa64_sme2, do_movt, a, tcg_gen_st_i64)
399
400	static bool trans_MOVT_ztz(DisasContext s, arg_MOVT_ztz a)
401	{
402	if (!dc_isar_feature(aa64_sme_lutv2, s)) {
403	return false;
404	}
405	if (sme_sm_enabled_check(s) && sme2_zt0_enabled_check(s)) {
406	int svl = streaming_vec_reg_size(s);
407	int tsize = MIN(svl, 64);
408	int offset = (a->off % (64 / tsize)) * tsize;
409
410	tcg_gen_gvec_mov(MO_64,
411	offsetof(CPUARMState, za_state.zt0) + offset,
412	vec_full_reg_offset(s, a->rt), tsize,
413	offset ? tsize : 64);
414	}
415	return true;
416	}
417
418	static bool trans_LDST1(DisasContext s, arg_LDST1 a)
419	{
420	typedef void GenLdSt1(TCGv_env, TCGv_ptr, TCGv_ptr, TCGv, TCGv_i64);
421
422	/*
423	* Indexed by [esz][be][v][mte][st], which is (except for load/store)
424	* also the order in which the elements appear in the function names,
425	* and so how we must concatenate the pieces.
426	*/
427
428	#define FN_LS(F) { gen_helper_sme_ld1##F, gen_helper_sme_st1##F }
429	#define FN_MTE(F) { FN_LS(F), FN_LS(F##_mte) }
430	#define FN_HV(F) { FN_MTE(F##_h), FN_MTE(F##_v) }
431	#define FN_END(L, B) { FN_HV(L), FN_HV(B) }
432
433	static GenLdSt1 * const fns[5][2][2][2][2] = {
434	FN_END(b, b),
435	FN_END(h_le, h_be),
436	FN_END(s_le, s_be),
437	FN_END(d_le, d_be),
438	FN_END(q_le, q_be),
439	};
440
441	#undef FN_LS
442	#undef FN_MTE
443	#undef FN_HV
444	#undef FN_END
445
446	TCGv_ptr t_za, t_pg;
447	TCGv_i64 addr;
448	uint64_t desc;
449	bool be = s->be_data == MO_BE;
450	bool mte = s->mte_active[0];
451
452	if (!dc_isar_feature(aa64_sme, s)) {
453	return false;
454	}
455	if (!sme_smza_enabled_check(s)) {
456	return true;
457	}
458
459	t_za = get_tile_rowcol(s, a->esz, a->rs, a->za, a->off, 1, 0, a->v);
460	t_pg = pred_full_reg_ptr(s, a->pg);
461	addr = tcg_temp_new_i64();
462
463	tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->esz);
464	tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
465
466	if (!mte) {
467	addr = clean_data_tbi(s, addr);
468	}
469
470	desc = make_svemte_desc(s, streaming_vec_reg_size(s), 1, a->esz, a->st, 0);
471
472	fns[a->esz][be][a->v][mte][a->st](tcg_env, t_za, t_pg, addr,
473	tcg_constant_i64(desc));
474	return true;
475	}
476
477	typedef void GenLdStR(DisasContext *, TCGv_ptr, int, int, int, int, MemOp);
478
479	static bool do_ldst_r(DisasContext s, arg_ldstr a, GenLdStR *fn)
480	{
481	if (sme_za_enabled_check(s)) {
482	int svl = streaming_vec_reg_size(s);
483	int imm = a->imm;
484	TCGv_ptr base = get_zarray(s, a->rv, imm, 1, 0);
485
486	fn(s, base, 0, svl, a->rn, imm * svl,
487	s->align_mem ? MO_ALIGN_16 : MO_UNALN);
488	}
489	return true;
490	}
491
492	TRANS_FEAT(LDR, aa64_sme, do_ldst_r, a, gen_sve_ldr)
493	TRANS_FEAT(STR, aa64_sme, do_ldst_r, a, gen_sve_str)
494
495	static bool do_ldst_zt0(DisasContext s, arg_ldstzt0 a, GenLdStR *fn)
496	{
497	if (sme2_zt0_enabled_check(s)) {
498	fn(s, tcg_env, offsetof(CPUARMState, za_state.zt0),
499	sizeof_field(CPUARMState, za_state.zt0), a->rn, 0,
500	s->align_mem ? MO_ALIGN_16 : MO_UNALN);
501	}
502	return true;
503	}
504
505	TRANS_FEAT(LDR_zt0, aa64_sme2, do_ldst_zt0, a, gen_sve_ldr)
506	TRANS_FEAT(STR_zt0, aa64_sme2, do_ldst_zt0, a, gen_sve_str)
507
508	static bool do_adda(DisasContext s, arg_adda a, MemOp esz,
509	gen_helper_gvec_4 *fn)
510	{
511	int svl = streaming_vec_reg_size(s);
512	uint32_t desc = simd_desc(svl, svl, 0);
513	TCGv_ptr za, zn, pn, pm;
514
515	if (!sme_smza_enabled_check(s)) {
516	return true;
517	}
518
519	za = get_tile(s, esz, a->zad);
520	zn = vec_full_reg_ptr(s, a->zn);
521	pn = pred_full_reg_ptr(s, a->pn);
522	pm = pred_full_reg_ptr(s, a->pm);
523
524	fn(za, zn, pn, pm, tcg_constant_i32(desc));
525	return true;
526	}
527
528	TRANS_FEAT(ADDHA_s, aa64_sme, do_adda, a, MO_32, gen_helper_sme_addha_s)
529	TRANS_FEAT(ADDVA_s, aa64_sme, do_adda, a, MO_32, gen_helper_sme_addva_s)
530	TRANS_FEAT(ADDHA_d, aa64_sme_i16i64, do_adda, a, MO_64, gen_helper_sme_addha_d)
531	TRANS_FEAT(ADDVA_d, aa64_sme_i16i64, do_adda, a, MO_64, gen_helper_sme_addva_d)
532
533	static bool do_outprod(DisasContext s, arg_op a, MemOp esz,
534	gen_helper_gvec_5 *fn)
535	{
536	int svl = streaming_vec_reg_size(s);
537	uint32_t desc = simd_desc(svl, svl, a->sub);
538	TCGv_ptr za, zn, zm, pn, pm;
539
540	if (!sme_smza_enabled_check(s)) {
541	return true;
542	}
543
544	za = get_tile(s, esz, a->zad);
545	zn = vec_full_reg_ptr(s, a->zn);
546	zm = vec_full_reg_ptr(s, a->zm);
547	pn = pred_full_reg_ptr(s, a->pn);
548	pm = pred_full_reg_ptr(s, a->pm);
549
550	fn(za, zn, zm, pn, pm, tcg_constant_i32(desc));
551	return true;
552	}
553
554	static bool do_outprod_fpst(DisasContext s, arg_op a, MemOp esz,
555	ARMFPStatusFlavour e_fpst,
556	gen_helper_gvec_5_ptr *fn)
557	{
558	int svl = streaming_vec_reg_size(s);
559	uint32_t desc = simd_desc(svl, svl, 0);
560	TCGv_ptr za, zn, zm, pn, pm, fpst;
561
562	if (!sme_smza_enabled_check(s)) {
563	return true;
564	}
565
566	za = get_tile(s, esz, a->zad);
567	zn = vec_full_reg_ptr(s, a->zn);
568	zm = vec_full_reg_ptr(s, a->zm);
569	pn = pred_full_reg_ptr(s, a->pn);
570	pm = pred_full_reg_ptr(s, a->pm);
571	fpst = fpstatus_ptr(e_fpst);
572
573	fn(za, zn, zm, pn, pm, fpst, tcg_constant_i32(desc));
574	return true;
575	}
576
577	static bool do_outprod_env(DisasContext s, arg_op a, MemOp esz,
578	gen_helper_gvec_5_ptr *fn)
579	{
580	int svl = streaming_vec_reg_size(s);
581	uint32_t desc = simd_desc(svl, svl, 0);
582	TCGv_ptr za, zn, zm, pn, pm;
583
584	if (!sme_smza_enabled_check(s)) {
585	return true;
586	}
587
588	za = get_tile(s, esz, a->zad);
589	zn = vec_full_reg_ptr(s, a->zn);
590	zm = vec_full_reg_ptr(s, a->zm);
591	pn = pred_full_reg_ptr(s, a->pn);
592	pm = pred_full_reg_ptr(s, a->pm);
593
594	fn(za, zn, zm, pn, pm, tcg_env, tcg_constant_i32(desc));
595	return true;
596	}
597
598	TRANS_FEAT(FMOPA_w_h, aa64_sme, do_outprod_env, a, MO_32,
599	!a->sub ? gen_helper_sme_fmopa_w_h
600	: !s->fpcr_ah ? gen_helper_sme_fmops_w_h
601	: gen_helper_sme_ah_fmops_w_h)
602	TRANS_FEAT(FMOPA_h, aa64_sme_f16f16, do_outprod_fpst, a, MO_16, FPST_ZA_F16,
603	!a->sub ? gen_helper_sme_fmopa_h
604	: !s->fpcr_ah ? gen_helper_sme_fmops_h
605	: gen_helper_sme_ah_fmops_h)
606	TRANS_FEAT(FMOPA_s, aa64_sme, do_outprod_fpst, a, MO_32, FPST_ZA,
607	!a->sub ? gen_helper_sme_fmopa_s
608	: !s->fpcr_ah ? gen_helper_sme_fmops_s
609	: gen_helper_sme_ah_fmops_s)
610	TRANS_FEAT(FMOPA_d, aa64_sme_f64f64, do_outprod_fpst, a, MO_64, FPST_ZA,
611	!a->sub ? gen_helper_sme_fmopa_d
612	: !s->fpcr_ah ? gen_helper_sme_fmops_d
613	: gen_helper_sme_ah_fmops_d)
614
615	TRANS_FEAT(BFMOPA, aa64_sme_b16b16, do_outprod_fpst, a, MO_16, FPST_ZA,
616	!a->sub ? gen_helper_sme_bfmopa
617	: !s->fpcr_ah ? gen_helper_sme_bfmops
618	: gen_helper_sme_ah_bfmops)
619
620	TRANS_FEAT(BFMOPA_w, aa64_sme, do_outprod_env, a, MO_32,
621	!a->sub ? gen_helper_sme_bfmopa_w
622	: !s->fpcr_ah ? gen_helper_sme_bfmops_w
623	: gen_helper_sme_ah_bfmops_w)
624
625	static bool do_outprod_fp8(DisasContext s, arg_op a, MemOp esz,
626	gen_helper_gvec_5_ptr *fn)
627	{
628	if (fpmr_access_check(s) && sme_smza_enabled_check(s)) {
629	int svl = streaming_vec_reg_size(s);
630	uint32_t desc = simd_desc(svl, svl, 0);
631
632	TCGv_ptr za = get_tile(s, esz, a->zad);
633	TCGv_ptr zn = vec_full_reg_ptr(s, a->zn);
634	TCGv_ptr zm = vec_full_reg_ptr(s, a->zm);
635	TCGv_ptr pn = pred_full_reg_ptr(s, a->pn);
636	TCGv_ptr pm = pred_full_reg_ptr(s, a->pm);
637
638	fn(za, zn, zm, pn, pm, tcg_env, tcg_constant_i32(desc));
639	}
640	return true;
641	}
642
643	TRANS_FEAT(FMOPA_sb, aa64_sme_f8f32, do_outprod_fp8,
644	a, MO_32, gen_helper_sme_fmopa_sb)
645	TRANS_FEAT(FMOPA_hb, aa64_sme_f8f16, do_outprod_fp8,
646	a, MO_16, gen_helper_sme_fmopa_hb)
647
648	TRANS_FEAT(SMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_smopa_s)
649	TRANS_FEAT(UMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_umopa_s)
650	TRANS_FEAT(SUMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_sumopa_s)
651	TRANS_FEAT(USMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_usmopa_s)
652
653	TRANS_FEAT(SMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_smopa_d)
654	TRANS_FEAT(UMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_umopa_d)
655	TRANS_FEAT(SUMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_sumopa_d)
656	TRANS_FEAT(USMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_usmopa_d)
657
658	TRANS_FEAT(BMOPA, aa64_sme2, do_outprod, a, MO_32, gen_helper_sme2_bmopa_s)
659	TRANS_FEAT(SMOPA2_s, aa64_sme2, do_outprod, a, MO_32, gen_helper_sme2_smopa2_s)
660	TRANS_FEAT(UMOPA2_s, aa64_sme2, do_outprod, a, MO_32, gen_helper_sme2_umopa2_s)
661
662	static bool do_zzz_n1(DisasContext s, arg_zzz_en a, GVecGen3Fn *fn)
663	{
664	int esz = a->esz, vsz, mofs;
665	int overlap = -1;
666
667	if (!sme_sm_enabled_check(s)) {
668	return true;
669	}
670
671	mofs = vec_full_reg_offset(s, a->zm);
672	vsz = streaming_vec_reg_size(s);
673
674	for (int i = 0, n = a->n; i < n; i++) {
675	int dofs = vec_full_reg_offset(s, a->zd + i);
676	int nofs = vec_full_reg_offset(s, a->zn + i);
677	if (dofs == mofs) {
678	overlap = i;
679	} else {
680	fn(esz, dofs, nofs, mofs, vsz, vsz);
681	}
682	}
683	if (overlap >= 0) {
684	int nofs = vec_full_reg_offset(s, a->zn + overlap);
685	fn(esz, mofs, nofs, mofs, vsz, vsz);
686	}
687	return true;
688	}
689
690	static void gen_sme2_srshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
691	uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
692	{
693	static gen_helper_gvec_3 * const fns[] = {
694	gen_helper_gvec_srshl_b, gen_helper_sme2_srshl_h,
695	gen_helper_sme2_srshl_s, gen_helper_sme2_srshl_d,
696	};
697	tcg_debug_assert(vece <= MO_64);
698	tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
699	}
700
701	static void gen_sme2_urshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
702	uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
703	{
704	static gen_helper_gvec_3 * const fns[] = {
705	gen_helper_gvec_urshl_b, gen_helper_sme2_urshl_h,
706	gen_helper_sme2_urshl_s, gen_helper_sme2_urshl_d,
707	};
708	tcg_debug_assert(vece <= MO_64);
709	tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
710	}
711
712	TRANS_FEAT(ADD_n1, aa64_sme2, do_zzz_n1, a, tcg_gen_gvec_add)
713	TRANS_FEAT(SMAX_n1, aa64_sme2, do_zzz_n1, a, tcg_gen_gvec_smax)
714	TRANS_FEAT(SMIN_n1, aa64_sme2, do_zzz_n1, a, tcg_gen_gvec_smin)
715	TRANS_FEAT(UMAX_n1, aa64_sme2, do_zzz_n1, a, tcg_gen_gvec_umax)
716	TRANS_FEAT(UMIN_n1, aa64_sme2, do_zzz_n1, a, tcg_gen_gvec_umin)
717	TRANS_FEAT(SRSHL_n1, aa64_sme2, do_zzz_n1, a, gen_sme2_srshl)
718	TRANS_FEAT(URSHL_n1, aa64_sme2, do_zzz_n1, a, gen_sme2_urshl)
719	TRANS_FEAT(SQDMULH_n1, aa64_sme2, do_zzz_n1, a, gen_gvec_sve2_sqdmulh)
720
721	static bool do_zzz_nn(DisasContext s, arg_zzz_en a, GVecGen3Fn *fn)
722	{
723	int esz = a->esz, vsz;
724
725	if (!sme_sm_enabled_check(s)) {
726	return true;
727	}
728
729	vsz = streaming_vec_reg_size(s);
730
731	for (int i = 0, n = a->n; i < n; i++) {
732	int dofs = vec_full_reg_offset(s, a->zd + i);
733	int nofs = vec_full_reg_offset(s, a->zn + i);
734	int mofs = vec_full_reg_offset(s, a->zm + i);
735
736	fn(esz, dofs, nofs, mofs, vsz, vsz);
737	}
738	return true;
739	}
740
741	TRANS_FEAT(SMAX_nn, aa64_sme2, do_zzz_nn, a, tcg_gen_gvec_smax)
742	TRANS_FEAT(SMIN_nn, aa64_sme2, do_zzz_nn, a, tcg_gen_gvec_smin)
743	TRANS_FEAT(UMAX_nn, aa64_sme2, do_zzz_nn, a, tcg_gen_gvec_umax)
744	TRANS_FEAT(UMIN_nn, aa64_sme2, do_zzz_nn, a, tcg_gen_gvec_umin)
745	TRANS_FEAT(SRSHL_nn, aa64_sme2, do_zzz_nn, a, gen_sme2_srshl)
746	TRANS_FEAT(URSHL_nn, aa64_sme2, do_zzz_nn, a, gen_sme2_urshl)
747	TRANS_FEAT(SQDMULH_nn, aa64_sme2, do_zzz_nn, a, gen_gvec_sve2_sqdmulh)
748
749	static bool do_zzz_n1_fpst(DisasContext s, arg_zzz_en a,
750	gen_helper_gvec_3_ptr *fn)
751	{
752	int esz = a->esz, vsz, mofs;
753	int overlap = -1;
754	TCGv_ptr fpst;
755
756	if (fn == NULL) {
757	return false;
758	}
759	if (!sme_sm_enabled_check(s)) {
760	return true;
761	}
762
763	fpst = fpstatus_ptr(esz == MO_16 ? FPST_A64_F16 : FPST_A64);
764	mofs = vec_full_reg_offset(s, a->zm);
765	vsz = streaming_vec_reg_size(s);
766
767	for (int i = 0, n = a->n; i < n; i++) {
768	int dofs = vec_full_reg_offset(s, a->zd + i);
769	int nofs = vec_full_reg_offset(s, a->zn + i);
770	if (dofs == mofs) {
771	overlap = i;
772	} else {
773	tcg_gen_gvec_3_ptr(dofs, nofs, mofs, fpst, vsz, vsz, 0, fn);
774	}
775	}
776	if (overlap >= 0) {
777	int nofs = vec_full_reg_offset(s, a->zn + overlap);
778	tcg_gen_gvec_3_ptr(mofs, nofs, mofs, fpst, vsz, vsz, 0, fn);
779	}
780	return true;
781	}
782
783	static bool do_zzz_nn_fpst(DisasContext s, arg_zzz_en a,
784	gen_helper_gvec_3_ptr *fn)
785	{
786	int esz = a->esz, vsz;
787	TCGv_ptr fpst;
788
789	if (fn == NULL) {
790	return false;
791	}
792	if (!sme_sm_enabled_check(s)) {
793	return true;
794	}
795
796	fpst = fpstatus_ptr(esz == MO_16 ? FPST_A64_F16 : FPST_A64);
797	vsz = streaming_vec_reg_size(s);
798
799	for (int i = 0, n = a->n; i < n; i++) {
800	int dofs = vec_full_reg_offset(s, a->zd + i);
801	int nofs = vec_full_reg_offset(s, a->zn + i);
802	int mofs = vec_full_reg_offset(s, a->zm + i);
803
804	tcg_gen_gvec_3_ptr(dofs, nofs, mofs, fpst, vsz, vsz, 0, fn);
805	}
806	return true;
807	}
808
809	static gen_helper_gvec_3_ptr * const f_vector_fmax[2][4] = {
810	{ NULL,
811	gen_helper_gvec_fmax_h,
812	gen_helper_gvec_fmax_s,
813	gen_helper_gvec_fmax_d },
814	{ NULL,
815	gen_helper_gvec_ah_fmax_h,
816	gen_helper_gvec_ah_fmax_s,
817	gen_helper_gvec_ah_fmax_d },
818	};
819	TRANS_FEAT(FMAX_n1, aa64_sme2, do_zzz_n1_fpst, a, f_vector_fmax[s->fpcr_ah][a->esz])
820	TRANS_FEAT(FMAX_nn, aa64_sme2, do_zzz_nn_fpst, a, f_vector_fmax[s->fpcr_ah][a->esz])
821	TRANS_FEAT(BFMAX_n1, aa64_sme2_sve_b16b16, do_zzz_n1_fpst, a,
822	s->fpcr_ah ? gen_helper_gvec_ah_fmax_b16 : gen_helper_gvec_fmax_b16)
823	TRANS_FEAT(BFMAX_nn, aa64_sme2_sve_b16b16, do_zzz_nn_fpst, a,
824	s->fpcr_ah ? gen_helper_gvec_ah_fmax_b16 : gen_helper_gvec_fmax_b16)
825
826	static gen_helper_gvec_3_ptr * const f_vector_fmin[2][4] = {
827	{ NULL,
828	gen_helper_gvec_fmin_h,
829	gen_helper_gvec_fmin_s,
830	gen_helper_gvec_fmin_d },
831	{ NULL,
832	gen_helper_gvec_ah_fmin_h,
833	gen_helper_gvec_ah_fmin_s,
834	gen_helper_gvec_ah_fmin_d },
835	};
836	TRANS_FEAT(FMIN_n1, aa64_sme2, do_zzz_n1_fpst, a, f_vector_fmin[s->fpcr_ah][a->esz])
837	TRANS_FEAT(FMIN_nn, aa64_sme2, do_zzz_nn_fpst, a, f_vector_fmin[s->fpcr_ah][a->esz])
838	TRANS_FEAT(BFMIN_n1, aa64_sme2_sve_b16b16, do_zzz_n1_fpst, a,
839	s->fpcr_ah ? gen_helper_gvec_ah_fmin_b16 : gen_helper_gvec_fmin_b16)
840	TRANS_FEAT(BFMIN_nn, aa64_sme2_sve_b16b16, do_zzz_nn_fpst, a,
841	s->fpcr_ah ? gen_helper_gvec_ah_fmin_b16 : gen_helper_gvec_fmin_b16)
842
843	static gen_helper_gvec_3_ptr * const f_vector_fmaxnm[4] = {
844	NULL,
845	gen_helper_gvec_fmaxnum_h,
846	gen_helper_gvec_fmaxnum_s,
847	gen_helper_gvec_fmaxnum_d,
848	};
849	TRANS_FEAT(FMAXNM_n1, aa64_sme2, do_zzz_n1_fpst, a, f_vector_fmaxnm[a->esz])
850	TRANS_FEAT(FMAXNM_nn, aa64_sme2, do_zzz_nn_fpst, a, f_vector_fmaxnm[a->esz])
851	TRANS_FEAT(BFMAXNM_n1, aa64_sme2_sve_b16b16, do_zzz_n1_fpst, a,
852	gen_helper_gvec_fmaxnum_b16)
853	TRANS_FEAT(BFMAXNM_nn, aa64_sme2_sve_b16b16, do_zzz_nn_fpst, a,
854	gen_helper_gvec_fmaxnum_b16)
855
856	static gen_helper_gvec_3_ptr * const f_vector_fminnm[4] = {
857	NULL,
858	gen_helper_gvec_fminnum_h,
859	gen_helper_gvec_fminnum_s,
860	gen_helper_gvec_fminnum_d,
861	};
862	TRANS_FEAT(FMINNM_n1, aa64_sme2, do_zzz_n1_fpst, a, f_vector_fminnm[a->esz])
863	TRANS_FEAT(FMINNM_nn, aa64_sme2, do_zzz_nn_fpst, a, f_vector_fminnm[a->esz])
864	TRANS_FEAT(BFMINNM_n1, aa64_sme2_sve_b16b16, do_zzz_n1_fpst, a,
865	gen_helper_gvec_fminnum_b16)
866	TRANS_FEAT(BFMINNM_nn, aa64_sme2_sve_b16b16, do_zzz_nn_fpst, a,
867	gen_helper_gvec_fminnum_b16)
868
869	static gen_helper_gvec_3_ptr * const f_vector_famax[4] = {
870	NULL,
871	gen_helper_gvec_famax_h,
872	gen_helper_gvec_famax_s,
873	gen_helper_gvec_famax_d,
874	};
875	TRANS_FEAT(FAMAX_nn, aa64_sme2_faminmax, do_zzz_nn_fpst, a, f_vector_famax[a->esz])
876
877	static gen_helper_gvec_3_ptr * const f_vector_famin[4] = {
878	NULL,
879	gen_helper_gvec_famin_h,
880	gen_helper_gvec_famin_s,
881	gen_helper_gvec_famin_d,
882	};
883	TRANS_FEAT(FAMIN_nn, aa64_sme2_faminmax, do_zzz_nn_fpst, a, f_vector_famin[a->esz])
884
885	static gen_helper_gvec_3_ptr * const f_vector_fscale[4] = {
886	NULL,
887	gen_helper_gvec_fscale_h,
888	gen_helper_gvec_fscale_s,
889	gen_helper_gvec_fscale_d,
890	};
891	TRANS_FEAT(FSCALE_n1, aa64_sme2_f8cvt, do_zzz_n1_fpst, a, f_vector_fscale[a->esz])
892	TRANS_FEAT(FSCALE_nn, aa64_sme2_f8cvt, do_zzz_nn_fpst, a, f_vector_fscale[a->esz])
893	TRANS_FEAT(BFSCALE_n1, aa64_sme2_sve_bfscale, do_zzz_n1_fpst, a,
894	gen_helper_gvec_fscale_b16)
895	TRANS_FEAT(BFSCALE_nn, aa64_sme2_sve_bfscale, do_zzz_nn_fpst, a,
896	gen_helper_gvec_fscale_b16)
897
898	static gen_helper_gvec_3_ptr * const f_vector_fmul[4] = {
899	NULL,
900	gen_helper_gvec_fmul_h,
901	gen_helper_gvec_fmul_s,
902	gen_helper_gvec_fmul_d,
903	};
904	TRANS_FEAT(FMUL_n1, aa64_sme2p2, do_zzz_n1_fpst, a, f_vector_fmul[a->esz])
905	TRANS_FEAT(FMUL_nn, aa64_sme2p2, do_zzz_nn_fpst, a, f_vector_fmul[a->esz])
906	TRANS_FEAT(BFMUL_n1, aa64_sme2_sve_bfscale, do_zzz_n1_fpst, a,
907	gen_helper_gvec_fmul_b16)
908	TRANS_FEAT(BFMUL_nn, aa64_sme2_sve_bfscale, do_zzz_nn_fpst, a,
909	gen_helper_gvec_fmul_b16)
910
911	/* Add/Sub vector Z[m] to each Z[nN] with result in ZA[dN]. */
912	static bool do_azz_n1(DisasContext s, arg_azz_n a, int esz,
913	GVecGen3FnVar *fn)
914	{
915	TCGv_ptr t_za;
916	int svl, n, o_zm;
917
918	if (!sme_smza_enabled_check(s)) {
919	return true;
920	}
921
922	n = a->n;
923	t_za = get_zarray(s, a->rv, a->off, n, 0);
924	o_zm = vec_full_reg_offset(s, a->zm);
925	svl = streaming_vec_reg_size(s);
926
927	for (int i = 0; i < n; ++i) {
928	int o_za = (svl / n * sizeof(ARMVectorReg)) * i;
929	int o_zn = vec_full_reg_offset(s, (a->zn + i) % 32);
930
931	fn(esz, t_za, o_za, tcg_env, o_zn, tcg_env, o_zm, svl, svl);
932	}
933	return true;
934	}
935
936	TRANS_FEAT(ADD_azz_n1_s, aa64_sme2, do_azz_n1, a, MO_32, tcg_gen_gvec_add_var)
937	TRANS_FEAT(SUB_azz_n1_s, aa64_sme2, do_azz_n1, a, MO_32, tcg_gen_gvec_sub_var)
938	TRANS_FEAT(ADD_azz_n1_d, aa64_sme2_i16i64, do_azz_n1, a, MO_64, tcg_gen_gvec_add_var)
939	TRANS_FEAT(SUB_azz_n1_d, aa64_sme2_i16i64, do_azz_n1, a, MO_64, tcg_gen_gvec_sub_var)
940
941	/* Add/Sub each vector Z[mN] to each Z[nN] with result in ZA[dN]. /
942	static bool do_azz_nn(DisasContext s, arg_azz_n a, int esz,
943	GVecGen3FnVar *fn)
944	{
945	TCGv_ptr t_za;
946	int svl, n;
947
948	if (!sme_smza_enabled_check(s)) {
949	return true;
950	}
951
952	n = a->n;
953	t_za = get_zarray(s, a->rv, a->off, n, 1);
954	svl = streaming_vec_reg_size(s);
955
956	for (int i = 0; i < n; ++i) {
957	int o_za = (svl / n * sizeof(ARMVectorReg)) * i;
958	int o_zn = vec_full_reg_offset(s, a->zn + i);
959	int o_zm = vec_full_reg_offset(s, a->zm + i);
960
961	fn(esz, t_za, o_za, tcg_env, o_zn, tcg_env, o_zm, svl, svl);
962	}
963	return true;
964	}
965
966	TRANS_FEAT(ADD_azz_nn_s, aa64_sme2, do_azz_nn, a, MO_32, tcg_gen_gvec_add_var)
967	TRANS_FEAT(SUB_azz_nn_s, aa64_sme2, do_azz_nn, a, MO_32, tcg_gen_gvec_sub_var)
968	TRANS_FEAT(ADD_azz_nn_d, aa64_sme2_i16i64, do_azz_nn, a, MO_64, tcg_gen_gvec_add_var)
969	TRANS_FEAT(SUB_azz_nn_d, aa64_sme2_i16i64, do_azz_nn, a, MO_64, tcg_gen_gvec_sub_var)
970
971	/* Add/Sub each ZA[dN] += Z[mN] */
972	static bool do_aaz(DisasContext s, arg_az_n a, int esz, GVecGen3FnVar *fn)
973	{
974	TCGv_ptr t_za;
975	int svl, n;
976
977	if (!sme_smza_enabled_check(s)) {
978	return true;
979	}
980
981	n = a->n;
982	t_za = get_zarray(s, a->rv, a->off, n, 0);
983	svl = streaming_vec_reg_size(s);
984
985	for (int i = 0; i < n; ++i) {
986	int o_za = (svl / n * sizeof(ARMVectorReg)) * i;
987	int o_zm = vec_full_reg_offset(s, a->zm + i);
988
989	fn(esz, t_za, o_za, t_za, o_za, tcg_env, o_zm, svl, svl);
990	}
991	return true;
992	}
993
994	TRANS_FEAT(ADD_aaz_s, aa64_sme2, do_aaz, a, MO_32, tcg_gen_gvec_add_var)
995	TRANS_FEAT(SUB_aaz_s, aa64_sme2, do_aaz, a, MO_32, tcg_gen_gvec_sub_var)
996	TRANS_FEAT(ADD_aaz_d, aa64_sme2_i16i64, do_aaz, a, MO_64, tcg_gen_gvec_add_var)
997	TRANS_FEAT(SUB_aaz_d, aa64_sme2_i16i64, do_aaz, a, MO_64, tcg_gen_gvec_sub_var)
998
999	/*
1000	* Expand array multi-vector single (n1), array multi-vector (nn),
1001	* and array multi-vector indexed (nx), for floating-point accumulate.
1002	* multi: true for nn, false for n1.
1003	* fpst: >= 0 to set ptr argument for FPST_*, < 0 for ENV.
1004	* data: stuff for simd_data, including any index.
1005	*/
1006	#define FPST_ENV -1
1007
1008	static bool do_azz_fp(DisasContext *s, int nreg, int nsel,
1009	int rv, int off, int zn, int zm,
1010	int data, int shsel, bool multi, int fpst,
1011	gen_helper_gvec_3_ptr *fn)
1012	{
1013	if (sme_smza_enabled_check(s)) {
1014	int svl = streaming_vec_reg_size(s);
1015	int vstride = svl / nreg;
1016	TCGv_ptr t_za = get_zarray(s, rv, off, nreg, nsel);
1017	TCGv_ptr t, ptr;
1018
1019	if (fpst >= 0) {
1020	ptr = fpstatus_ptr(fpst);
1021	} else {
1022	ptr = tcg_env;
1023	}
1024	t = tcg_temp_new_ptr();
1025
1026	for (int r = 0; r < nreg; ++r) {
1027	TCGv_ptr t_zn = vec_full_reg_ptr(s, zn);
1028	TCGv_ptr t_zm = vec_full_reg_ptr(s, zm);
1029
1030	for (int i = 0; i < nsel; ++i) {
1031	int o_za = (r * vstride + i) * sizeof(ARMVectorReg);
1032	int desc = simd_desc(svl, svl, data \| (i << shsel));
1033
1034	tcg_gen_addi_ptr(t, t_za, o_za);
1035	fn(t, t_zn, t_zm, ptr, tcg_constant_i32(desc));
1036	}
1037
1038	/*
1039	* For multiple-and-single vectors, Zn may wrap.
1040	* For multiple vectors, both Zn and Zm are aligned.
1041	*/
1042	zn = (zn + 1) % 32;
1043	zm += multi;
1044	}
1045	}
1046	return true;
1047	}
1048
1049	static bool do_azz_acc_fp(DisasContext *s, int nreg, int nsel,
1050	int rv, int off, int zn, int zm,
1051	int data, int shsel, bool multi, int fpst,
1052	gen_helper_gvec_4_ptr *fn)
1053	{
1054	if (sme_smza_enabled_check(s)) {
1055	int svl = streaming_vec_reg_size(s);
1056	int vstride = svl / nreg;
1057	TCGv_ptr t_za = get_zarray(s, rv, off, nreg, nsel);
1058	TCGv_ptr t, ptr;
1059
1060	if (fpst >= 0) {
1061	ptr = fpstatus_ptr(fpst);
1062	} else {
1063	ptr = tcg_env;
1064	}
1065	t = tcg_temp_new_ptr();
1066
1067	for (int r = 0; r < nreg; ++r) {
1068	TCGv_ptr t_zn = vec_full_reg_ptr(s, zn);
1069	TCGv_ptr t_zm = vec_full_reg_ptr(s, zm);
1070
1071	for (int i = 0; i < nsel; ++i) {
1072	int o_za = (r * vstride + i) * sizeof(ARMVectorReg);
1073	int desc = simd_desc(svl, svl, data \| (i << shsel));
1074
1075	tcg_gen_addi_ptr(t, t_za, o_za);
1076	fn(t, t_zn, t_zm, t, ptr, tcg_constant_i32(desc));
1077	}
1078
1079	/*
1080	* For multiple-and-single vectors, Zn may wrap.
1081	* For multiple vectors, both Zn and Zm are aligned.
1082	*/
1083	zn = (zn + 1) % 32;
1084	zm += multi;
1085	}
1086	}
1087	return true;
1088	}
1089
1090	static bool do_azz_acc_fp8(DisasContext *s, int nreg, int nsel,
1091	int rv, int off, int zn, int zm,
1092	int data, int shsel, bool multi,
1093	gen_helper_gvec_3_ptr *fn)
1094	{
1095	/*
1096	* TODO: Could plausibly reuse do_azz_acc_fp, after the fpmr check,
1097	* but the fp8 helpers were written without a separate addend operand.
1098	*/
1099	if (fpmr_access_check(s) && sme_smza_enabled_check(s)) {
1100	int svl = streaming_vec_reg_size(s);
1101	int vstride = svl / nreg;
1102	TCGv_ptr t_za = get_zarray(s, rv, off, nreg, nsel);
1103	TCGv_ptr t;
1104
1105	t = tcg_temp_new_ptr();
1106
1107	for (int r = 0; r < nreg; ++r) {
1108	TCGv_ptr t_zn = vec_full_reg_ptr(s, zn);
1109	TCGv_ptr t_zm = vec_full_reg_ptr(s, zm);
1110
1111	for (int i = 0; i < nsel; ++i) {
1112	int o_za = (r * vstride + i) * sizeof(ARMVectorReg);
1113	int desc = simd_desc(svl, svl, data \| (i << shsel));
1114
1115	tcg_gen_addi_ptr(t, t_za, o_za);
1116	fn(t, t_zn, t_zm, tcg_env, tcg_constant_i32(desc));
1117	}
1118
1119	/*
1120	* For multiple-and-single vectors, Zn may wrap.
1121	* For multiple vectors, both Zn and Zm are aligned.
1122	*/
1123	zn = (zn + 1) % 32;
1124	zm += multi;
1125	}
1126	}
1127	return true;
1128	}
1129
1130
1131	static bool do_fmlal(DisasContext s, arg_azz_n a, bool sub, bool multi)
1132	{
1133	return do_azz_acc_fp(s, a->n, 2, a->rv, a->off, a->zn, a->zm,
1134	(1 << 2) \| sub, 1,
1135	multi, FPST_ENV, gen_helper_sve2_fmlal_zzzw_s);
1136	}
1137
1138	TRANS_FEAT(FMLAL_n1_sh, aa64_sme2, do_fmlal, a, false, false)
1139	TRANS_FEAT(FMLSL_n1_sh, aa64_sme2, do_fmlal, a, true, false)
1140	TRANS_FEAT(FMLAL_nn_sh, aa64_sme2, do_fmlal, a, false, true)
1141	TRANS_FEAT(FMLSL_nn_sh, aa64_sme2, do_fmlal, a, true, true)
1142
1143	static bool do_fmlall_fp8(DisasContext s, arg_azz_n a, bool multi)
1144	{
1145	return do_azz_acc_fp8(s, a->n, 4, a->rv, a->off, a->zn, a->zm,
1146	0, 0, multi, gen_helper_gvec_fmla_sb);
1147	}
1148
1149	TRANS_FEAT(FMLALL_n1_b, aa64_sme_f8f32, do_fmlall_fp8, a, false)
1150	TRANS_FEAT(FMLALL_nn_b, aa64_sme_f8f32, do_fmlall_fp8, a, true)
1151
1152	static bool do_fmlal_fp8(DisasContext s, arg_azz_n a, bool multi)
1153	{
1154	return do_azz_acc_fp8(s, a->n, 2, a->rv, a->off, a->zn, a->zm,
1155	0, 0, multi, gen_helper_gvec_fmla_hb);
1156	}
1157
1158	TRANS_FEAT(FMLAL_n1_hb, aa64_sme_f8f16, do_fmlal_fp8, a, false)
1159	TRANS_FEAT(FMLAL_nn_hb, aa64_sme_f8f16, do_fmlal_fp8, a, true)
1160
1161	static bool do_fmlal_nx(DisasContext s, arg_azx_n a, bool sub)
1162	{
1163	return do_azz_acc_fp(s, a->n, 2, a->rv, a->off, a->zn, a->zm,
1164	(a->idx << 3) \| (1 << 2) \| sub, 1,
1165	false, FPST_ENV, gen_helper_sve2_fmlal_zzxw_s);
1166	}
1167
1168	TRANS_FEAT(FMLAL_nx_sh, aa64_sme2, do_fmlal_nx, a, false)
1169	TRANS_FEAT(FMLSL_nx_sh, aa64_sme2, do_fmlal_nx, a, true)
1170
1171	TRANS_FEAT(FMLAL_nx_hb, aa64_sme_f8f16, do_azz_acc_fp8,
1172	a->n, 2, a->rv, a->off, a->zn, a->zm,
1173	a->idx << 2, 0, false, gen_helper_gvec_fmla_idx_hb)
1174
1175	static bool do_bfmlal(DisasContext s, arg_azz_n a, bool sub, bool multi)
1176	{
1177	return do_azz_acc_fp(s, a->n, 2, a->rv, a->off, a->zn, a->zm,
1178	0, 0, multi, FPST_ZA,
1179	(!sub ? gen_helper_gvec_bfmlal
1180	: s->fpcr_ah ? gen_helper_gvec_ah_bfmlsl
1181	: gen_helper_gvec_bfmlsl));
1182	}
1183
1184	TRANS_FEAT(BFMLAL_n1, aa64_sme2, do_bfmlal, a, false, false)
1185	TRANS_FEAT(BFMLSL_n1, aa64_sme2, do_bfmlal, a, true, false)
1186	TRANS_FEAT(BFMLAL_nn, aa64_sme2, do_bfmlal, a, false, true)
1187	TRANS_FEAT(BFMLSL_nn, aa64_sme2, do_bfmlal, a, true, true)
1188
1189	static bool do_bfmlal_nx(DisasContext s, arg_azx_n a, bool sub)
1190	{
1191	return do_azz_acc_fp(s, a->n, 2, a->rv, a->off, a->zn, a->zm,
1192	a->idx << 1, 0, false, FPST_ZA,
1193	!sub ? gen_helper_gvec_bfmlal_idx
1194	: s->fpcr_ah ? gen_helper_gvec_ah_bfmlsl_idx
1195	: gen_helper_gvec_bfmlsl_idx);
1196	}
1197
1198	TRANS_FEAT(BFMLAL_nx, aa64_sme2, do_bfmlal_nx, a, false)
1199	TRANS_FEAT(BFMLSL_nx, aa64_sme2, do_bfmlal_nx, a, true)
1200
1201	TRANS_FEAT(FMLALL_nx_b, aa64_sme_f8f32, do_azz_acc_fp8,
1202	a->n, 4, a->rv, a->off, a->zn, a->zm,
1203	a->idx << 2, 0, false, gen_helper_gvec_fmla_idx_sb)
1204
1205	TRANS_FEAT(FDOT_nx_b, aa64_sme_f8f32, do_azz_acc_fp8,
1206	a->n, 1, a->rv, a->off, a->zn, a->zm,
1207	a->idx, 0, false, gen_helper_gvec_fdot_idx_sb)
1208
1209	TRANS_FEAT(FDOT_nx_hb, aa64_sme_f8f16, do_azz_acc_fp8,
1210	a->n, 1, a->rv, a->off, a->zn, a->zm,
1211	a->idx, 0, false, gen_helper_gvec_fdot_idx_hb)
1212
1213	static bool do_fdot(DisasContext s, arg_azz_n a, bool multi)
1214	{
1215	return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 1, 0,
1216	multi, FPST_ENV, gen_helper_sme2_fdot_h);
1217	}
1218
1219	TRANS_FEAT(FDOT_n1, aa64_sme2, do_fdot, a, false)
1220	TRANS_FEAT(FDOT_nn, aa64_sme2, do_fdot, a, true)
1221
1222	static bool do_fdot_fp8(DisasContext s, arg_azz_n a, bool multi)
1223	{
1224	return do_azz_acc_fp8(s, a->n, 1, a->rv, a->off, a->zn, a->zm,
1225	0, 0, multi, gen_helper_gvec_fdot_sb);
1226	}
1227
1228	TRANS_FEAT(FDOT_n1_sb, aa64_sme_f8f32, do_fdot_fp8, a, false)
1229	TRANS_FEAT(FDOT_nn_sb, aa64_sme_f8f32, do_fdot_fp8, a, true)
1230
1231	static bool do_fdot_hb(DisasContext s, arg_azz_n a, bool multi)
1232	{
1233	return do_azz_acc_fp8(s, a->n, 1, a->rv, a->off, a->zn, a->zm,
1234	0, 0, multi, gen_helper_gvec_fdot_hb);
1235	}
1236
1237	TRANS_FEAT(FDOT_n1_hb, aa64_sme_f8f16, do_fdot_hb, a, false)
1238	TRANS_FEAT(FDOT_nn_hb, aa64_sme_f8f16, do_fdot_hb, a, true)
1239
1240	static bool do_fdot_nx(DisasContext s, arg_azx_n a)
1241	{
1242	return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm,
1243	a->idx \| (1 << 2), 0, false, FPST_ENV,
1244	gen_helper_sme2_fdot_idx_h);
1245	}
1246
1247	TRANS_FEAT(FDOT_nx, aa64_sme2, do_fdot_nx, a)
1248
1249	static bool do_bfdot(DisasContext s, arg_azz_n a, bool multi)
1250	{
1251	return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 0, 0,
1252	multi, FPST_ENV, gen_helper_gvec_bfdot);
1253	}
1254
1255	TRANS_FEAT(BFDOT_n1, aa64_sme2, do_bfdot, a, false)
1256	TRANS_FEAT(BFDOT_nn, aa64_sme2, do_bfdot, a, true)
1257
1258	static bool do_bfdot_nx(DisasContext s, arg_azx_n a)
1259	{
1260	return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, a->idx, 0,
1261	false, FPST_ENV, gen_helper_gvec_bfdot_idx);
1262	}
1263
1264	TRANS_FEAT(BFDOT_nx, aa64_sme2, do_bfdot_nx, a)
1265
1266	static bool do_vdot(DisasContext s, arg_azx_n a, gen_helper_gvec_4_ptr *fn)
1267	{
1268	if (sme_smza_enabled_check(s)) {
1269	int svl = streaming_vec_reg_size(s);
1270	int vstride = svl / 2;
1271	TCGv_ptr t_za = get_zarray(s, a->rv, a->off, 2, 1);
1272	TCGv_ptr t_zn = vec_full_reg_ptr(s, a->zn);
1273	TCGv_ptr t_zm = vec_full_reg_ptr(s, a->zm);
1274	TCGv_ptr t = tcg_temp_new_ptr();
1275
1276	for (int i = 0; i < 2; ++i) {
1277	int o_za = i * vstride * sizeof(ARMVectorReg);
1278	int desc = simd_desc(svl, svl, a->idx \| (i << 2));
1279
1280	tcg_gen_addi_ptr(t, t_za, o_za);
1281	fn(t, t_zn, t_zm, t, tcg_env, tcg_constant_i32(desc));
1282	}
1283	}
1284	return true;
1285	}
1286
1287	TRANS_FEAT(FVDOT_sh, aa64_sme, do_vdot, a, gen_helper_sme2_fvdot_idx_h)
1288	TRANS_FEAT(BFVDOT, aa64_sme, do_vdot, a, gen_helper_sme2_bfvdot_idx)
1289
1290	static bool do_fvdot_sb(DisasContext s, arg_azx_n a, bool top)
1291	{
1292	return do_azz_acc_fp8(s, a->n, 1, a->rv, a->off, a->zn, a->zm,
1293	(2 * a->idx + top) << 2, 0, false,
1294	gen_helper_sme_fvdot_idx_sb);
1295	}
1296
1297	TRANS_FEAT(FVDOTB_sb, aa64_sme_f8f32, do_fvdot_sb, a, false)
1298	TRANS_FEAT(FVDOTT_sb, aa64_sme_f8f32, do_fvdot_sb, a, true)
1299
1300	TRANS_FEAT(FVDOT_hb, aa64_sme_f8f16, do_azz_acc_fp8,
1301	a->n, 2, a->rv, a->off, a->zn, a->zm,
1302	(a->idx << 1), 0, false, gen_helper_sme_fvdot_idx_hb)
1303
1304	static bool do_fmla(DisasContext s, arg_azz_n a, bool multi,
1305	ARMFPStatusFlavour fpst, gen_helper_gvec_3_ptr *fn)
1306	{
1307	return do_azz_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm,
1308	0, 0, multi, fpst, fn);
1309	}
1310
1311	TRANS_FEAT(FMLA_n1_h, aa64_sme_f16f16, do_fmla, a, false, FPST_ZA_F16,
1312	gen_helper_gvec_vfma_h)
1313	TRANS_FEAT(FMLS_n1_h, aa64_sme_f16f16, do_fmla, a, false, FPST_ZA_F16,
1314	s->fpcr_ah ? gen_helper_gvec_ah_vfms_h : gen_helper_gvec_vfms_h)
1315	TRANS_FEAT(FMLA_nn_h, aa64_sme_f16f16, do_fmla, a, true, FPST_ZA_F16,
1316	gen_helper_gvec_vfma_h)
1317	TRANS_FEAT(FMLS_nn_h, aa64_sme_f16f16, do_fmla, a, true, FPST_ZA_F16,
1318	s->fpcr_ah ? gen_helper_gvec_ah_vfms_h : gen_helper_gvec_vfms_h)
1319
1320	TRANS_FEAT(FMLA_n1_s, aa64_sme2, do_fmla, a, false, FPST_ZA,
1321	gen_helper_gvec_vfma_s)
1322	TRANS_FEAT(FMLS_n1_s, aa64_sme2, do_fmla, a, false, FPST_ZA,
1323	s->fpcr_ah ? gen_helper_gvec_ah_vfms_s : gen_helper_gvec_vfms_s)
1324	TRANS_FEAT(FMLA_nn_s, aa64_sme2, do_fmla, a, true, FPST_ZA,
1325	gen_helper_gvec_vfma_s)
1326	TRANS_FEAT(FMLS_nn_s, aa64_sme2, do_fmla, a, true, FPST_ZA,
1327	s->fpcr_ah ? gen_helper_gvec_ah_vfms_s : gen_helper_gvec_vfms_s)
1328
1329	TRANS_FEAT(FMLA_n1_d, aa64_sme2_f64f64, do_fmla, a, false, FPST_ZA,
1330	gen_helper_gvec_vfma_d)
1331	TRANS_FEAT(FMLS_n1_d, aa64_sme2_f64f64, do_fmla, a, false, FPST_ZA,
1332	s->fpcr_ah ? gen_helper_gvec_ah_vfms_d : gen_helper_gvec_vfms_d)
1333	TRANS_FEAT(FMLA_nn_d, aa64_sme2_f64f64, do_fmla, a, true, FPST_ZA,
1334	gen_helper_gvec_vfma_d)
1335	TRANS_FEAT(FMLS_nn_d, aa64_sme2_f64f64, do_fmla, a, true, FPST_ZA,
1336	s->fpcr_ah ? gen_helper_gvec_ah_vfms_d : gen_helper_gvec_vfms_d)
1337
1338	TRANS_FEAT(BFMLA_n1, aa64_sme_b16b16, do_fmla, a, false, FPST_ZA,
1339	gen_helper_gvec_bfmla)
1340	TRANS_FEAT(BFMLS_n1, aa64_sme_b16b16, do_fmla, a, false, FPST_ZA,
1341	s->fpcr_ah ? gen_helper_gvec_ah_bfmls : gen_helper_gvec_bfmls)
1342	TRANS_FEAT(BFMLA_nn, aa64_sme_b16b16, do_fmla, a, true, FPST_ZA,
1343	gen_helper_gvec_bfmla)
1344	TRANS_FEAT(BFMLS_nn, aa64_sme_b16b16, do_fmla, a, true, FPST_ZA,
1345	s->fpcr_ah ? gen_helper_gvec_ah_bfmls : gen_helper_gvec_bfmls)
1346
1347	static bool do_fmla_nx(DisasContext s, arg_azx_n a,
1348	ARMFPStatusFlavour fpst, gen_helper_gvec_4_ptr *fn)
1349	{
1350	return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm,
1351	a->idx, 0, false, fpst, fn);
1352	}
1353
1354	TRANS_FEAT(FMLA_nx_h, aa64_sme_f16f16, do_fmla_nx, a, FPST_ZA_F16,
1355	gen_helper_gvec_fmla_idx_h)
1356	TRANS_FEAT(FMLS_nx_h, aa64_sme_f16f16, do_fmla_nx, a, FPST_ZA_F16,
1357	s->fpcr_ah ? gen_helper_gvec_ah_fmls_idx_h : gen_helper_gvec_fmls_idx_h)
1358	TRANS_FEAT(FMLA_nx_s, aa64_sme2, do_fmla_nx, a, FPST_ZA,
1359	gen_helper_gvec_fmla_idx_s)
1360	TRANS_FEAT(FMLS_nx_s, aa64_sme2, do_fmla_nx, a, FPST_ZA,
1361	s->fpcr_ah ? gen_helper_gvec_ah_fmls_idx_s : gen_helper_gvec_fmls_idx_s)
1362	TRANS_FEAT(FMLA_nx_d, aa64_sme2_f64f64, do_fmla_nx, a, FPST_ZA,
1363	gen_helper_gvec_fmla_idx_d)
1364	TRANS_FEAT(FMLS_nx_d, aa64_sme2_f64f64, do_fmla_nx, a, FPST_ZA,
1365	s->fpcr_ah ? gen_helper_gvec_ah_fmls_idx_d : gen_helper_gvec_fmls_idx_d)
1366
1367	TRANS_FEAT(BFMLA_nx, aa64_sme_b16b16, do_fmla_nx, a, FPST_ZA,
1368	gen_helper_gvec_bfmla_idx)
1369	TRANS_FEAT(BFMLS_nx, aa64_sme_b16b16, do_fmla_nx, a, FPST_ZA,
1370	s->fpcr_ah ? gen_helper_gvec_ah_bfmls_idx : gen_helper_gvec_bfmls_idx)
1371
1372	static bool do_faddsub(DisasContext s, arg_az_n a, ARMFPStatusFlavour fpst,
1373	gen_helper_gvec_3_ptr *fn)
1374	{
1375	if (sme_smza_enabled_check(s)) {
1376	int svl = streaming_vec_reg_size(s);
1377	int n = a->n;
1378	int zm = a->zm;
1379	int vstride = svl / n;
1380	TCGv_ptr t_za = get_zarray(s, a->rv, a->off, n, 0);
1381	TCGv_ptr ptr = fpstatus_ptr(fpst);
1382	TCGv_ptr t = tcg_temp_new_ptr();
1383
1384	for (int r = 0; r < n; ++r) {
1385	TCGv_ptr t_zm = vec_full_reg_ptr(s, zm + r);
1386	int o_za = r * vstride * sizeof(ARMVectorReg);
1387	int desc = simd_desc(svl, svl, 0);
1388
1389	tcg_gen_addi_ptr(t, t_za, o_za);
1390	fn(t, t, t_zm, ptr, tcg_constant_i32(desc));
1391	}
1392	}
1393	return true;
1394	}
1395
1396	TRANS_FEAT(FADD_nn_h, aa64_sme_f16f16_or_f8f16, do_faddsub, a,
1397	FPST_ZA_F16, gen_helper_gvec_fadd_h)
1398	TRANS_FEAT(FSUB_nn_h, aa64_sme_f16f16_or_f8f16, do_faddsub, a,
1399	FPST_ZA_F16, gen_helper_gvec_fsub_h)
1400
1401	TRANS_FEAT(FADD_nn_s, aa64_sme2, do_faddsub, a,
1402	FPST_ZA, gen_helper_gvec_fadd_s)
1403	TRANS_FEAT(FSUB_nn_s, aa64_sme2, do_faddsub, a,
1404	FPST_ZA, gen_helper_gvec_fsub_s)
1405
1406	TRANS_FEAT(FADD_nn_d, aa64_sme2_f64f64, do_faddsub, a,
1407	FPST_ZA, gen_helper_gvec_fadd_d)
1408	TRANS_FEAT(FSUB_nn_d, aa64_sme2_f64f64, do_faddsub, a,
1409	FPST_ZA, gen_helper_gvec_fsub_d)
1410
1411	TRANS_FEAT(BFADD_nn, aa64_sme_b16b16, do_faddsub, a,
1412	FPST_ZA, gen_helper_gvec_bfadd)
1413	TRANS_FEAT(BFSUB_nn, aa64_sme_b16b16, do_faddsub, a,
1414	FPST_ZA, gen_helper_gvec_bfsub)
1415
1416	/*
1417	* Expand array multi-vector single (n1), array multi-vector (nn),
1418	* and array multi-vector indexed (nx), for integer accumulate.
1419	* multi: true for nn, false for n1.
1420	* data: stuff for simd_data, including any index.
1421	*/
1422	static bool do_azz_acc(DisasContext *s, int nreg, int nsel,
1423	int rv, int off, int zn, int zm,
1424	int data, int shsel, bool multi,
1425	gen_helper_gvec_4 *fn)
1426	{
1427	if (sme_smza_enabled_check(s)) {
1428	int svl = streaming_vec_reg_size(s);
1429	int vstride = svl / nreg;
1430	TCGv_ptr t_za = get_zarray(s, rv, off, nreg, nsel);
1431	TCGv_ptr t = tcg_temp_new_ptr();
1432
1433	for (int r = 0; r < nreg; ++r) {
1434	TCGv_ptr t_zn = vec_full_reg_ptr(s, zn);
1435	TCGv_ptr t_zm = vec_full_reg_ptr(s, zm);
1436
1437	for (int i = 0; i < nsel; ++i) {
1438	int o_za = (r * vstride + i) * sizeof(ARMVectorReg);
1439	int desc = simd_desc(svl, svl, data \| (i << shsel));
1440
1441	tcg_gen_addi_ptr(t, t_za, o_za);
1442	fn(t, t_zn, t_zm, t, tcg_constant_i32(desc));
1443	}
1444
1445	/*
1446	* For multiple-and-single vectors, Zn may wrap.
1447	* For multiple vectors, both Zn and Zm are aligned.
1448	*/
1449	zn = (zn + 1) % 32;
1450	zm += multi;
1451	}
1452	}
1453	return true;
1454	}
1455
1456	static bool do_dot(DisasContext s, arg_azz_n a, bool multi,
1457	gen_helper_gvec_4 *fn)
1458	{
1459	return do_azz_acc(s, a->n, 1, a->rv, a->off, a->zn, a->zm,
1460	0, 0, multi, fn);
1461	}
1462
1463	static void gen_helper_gvec_sudot_4b(TCGv_ptr d, TCGv_ptr n, TCGv_ptr m,
1464	TCGv_ptr a, TCGv_i32 desc)
1465	{
1466	gen_helper_gvec_usdot_4b(d, m, n, a, desc);
1467	}
1468
1469	TRANS_FEAT(USDOT_n1, aa64_sme2, do_dot, a, false, gen_helper_gvec_usdot_4b)
1470	TRANS_FEAT(SUDOT_n1, aa64_sme2, do_dot, a, false, gen_helper_gvec_sudot_4b)
1471	TRANS_FEAT(SDOT_n1_2h, aa64_sme2, do_dot, a, false, gen_helper_gvec_sdot_2h)
1472	TRANS_FEAT(UDOT_n1_2h, aa64_sme2, do_dot, a, false, gen_helper_gvec_udot_2h)
1473	TRANS_FEAT(SDOT_n1_4b, aa64_sme2, do_dot, a, false, gen_helper_gvec_sdot_4b)
1474	TRANS_FEAT(UDOT_n1_4b, aa64_sme2, do_dot, a, false, gen_helper_gvec_udot_4b)
1475	TRANS_FEAT(SDOT_n1_4h, aa64_sme2_i16i64, do_dot, a, false, gen_helper_gvec_sdot_4h)
1476	TRANS_FEAT(UDOT_n1_4h, aa64_sme2_i16i64, do_dot, a, false, gen_helper_gvec_udot_4h)
1477
1478	TRANS_FEAT(USDOT_nn, aa64_sme2, do_dot, a, true, gen_helper_gvec_usdot_4b)
1479	TRANS_FEAT(SDOT_nn_2h, aa64_sme2, do_dot, a, true, gen_helper_gvec_sdot_2h)
1480	TRANS_FEAT(UDOT_nn_2h, aa64_sme2, do_dot, a, true, gen_helper_gvec_udot_2h)
1481	TRANS_FEAT(SDOT_nn_4b, aa64_sme2, do_dot, a, true, gen_helper_gvec_sdot_4b)
1482	TRANS_FEAT(UDOT_nn_4b, aa64_sme2, do_dot, a, true, gen_helper_gvec_udot_4b)
1483	TRANS_FEAT(SDOT_nn_4h, aa64_sme2_i16i64, do_dot, a, true, gen_helper_gvec_sdot_4h)
1484	TRANS_FEAT(UDOT_nn_4h, aa64_sme2_i16i64, do_dot, a, true, gen_helper_gvec_udot_4h)
1485
1486	static bool do_dot_nx(DisasContext s, arg_azx_n a, gen_helper_gvec_4 *fn)
1487	{
1488	return do_azz_acc(s, a->n, 1, a->rv, a->off, a->zn, a->zm,
1489	a->idx, 0, false, fn);
1490	}
1491
1492	TRANS_FEAT(USDOT_nx, aa64_sme2, do_dot_nx, a, gen_helper_gvec_usdot_idx_4b)
1493	TRANS_FEAT(SUDOT_nx, aa64_sme2, do_dot_nx, a, gen_helper_gvec_sudot_idx_4b)
1494	TRANS_FEAT(SDOT_nx_2h, aa64_sme2, do_dot_nx, a, gen_helper_gvec_sdot_idx_2h)
1495	TRANS_FEAT(UDOT_nx_2h, aa64_sme2, do_dot_nx, a, gen_helper_gvec_udot_idx_2h)
1496	TRANS_FEAT(SDOT_nx_4b, aa64_sme2, do_dot_nx, a, gen_helper_gvec_sdot_idx_4b)
1497	TRANS_FEAT(UDOT_nx_4b, aa64_sme2, do_dot_nx, a, gen_helper_gvec_udot_idx_4b)
1498	TRANS_FEAT(SDOT_nx_4h, aa64_sme2_i16i64, do_dot_nx, a, gen_helper_gvec_sdot_idx_4h)
1499	TRANS_FEAT(UDOT_nx_4h, aa64_sme2_i16i64, do_dot_nx, a, gen_helper_gvec_udot_idx_4h)
1500
1501	static bool do_vdot_nx(DisasContext s, arg_azx_n a, gen_helper_gvec_3 *fn)
1502	{
1503	if (sme_smza_enabled_check(s)) {
1504	int svl = streaming_vec_reg_size(s);
1505	fn(get_zarray(s, a->rv, a->off, a->n, 0),
1506	vec_full_reg_ptr(s, a->zn),
1507	vec_full_reg_ptr(s, a->zm),
1508	tcg_constant_i32(simd_desc(svl, svl, a->idx)));
1509	}
1510	return true;
1511	}
1512
1513	TRANS_FEAT(SVDOT_nx_2h, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_svdot_idx_2h)
1514	TRANS_FEAT(SVDOT_nx_4b, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_svdot_idx_4b)
1515	TRANS_FEAT(SVDOT_nx_4h, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_svdot_idx_4h)
1516
1517	TRANS_FEAT(UVDOT_nx_2h, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_uvdot_idx_2h)
1518	TRANS_FEAT(UVDOT_nx_4b, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_uvdot_idx_4b)
1519	TRANS_FEAT(UVDOT_nx_4h, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_uvdot_idx_4h)
1520
1521	TRANS_FEAT(SUVDOT_nx_4b, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_suvdot_idx_4b)
1522	TRANS_FEAT(USVDOT_nx_4b, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_usvdot_idx_4b)
1523
1524	static bool do_smlal(DisasContext s, arg_azz_n a, bool multi,
1525	gen_helper_gvec_4 *fn)
1526	{
1527	return do_azz_acc(s, a->n, 2, a->rv, a->off, a->zn, a->zm,
1528	0, 0, multi, fn);
1529	}
1530
1531	TRANS_FEAT(SMLAL_n1, aa64_sme2, do_smlal, a, false, gen_helper_sve2_smlal_zzzw_s)
1532	TRANS_FEAT(SMLSL_n1, aa64_sme2, do_smlal, a, false, gen_helper_sve2_smlsl_zzzw_s)
1533	TRANS_FEAT(UMLAL_n1, aa64_sme2, do_smlal, a, false, gen_helper_sve2_umlal_zzzw_s)
1534	TRANS_FEAT(UMLSL_n1, aa64_sme2, do_smlal, a, false, gen_helper_sve2_umlsl_zzzw_s)
1535
1536	TRANS_FEAT(SMLAL_nn, aa64_sme2, do_smlal, a, true, gen_helper_sve2_smlal_zzzw_s)
1537	TRANS_FEAT(SMLSL_nn, aa64_sme2, do_smlal, a, true, gen_helper_sve2_smlsl_zzzw_s)
1538	TRANS_FEAT(UMLAL_nn, aa64_sme2, do_smlal, a, true, gen_helper_sve2_umlal_zzzw_s)
1539	TRANS_FEAT(UMLSL_nn, aa64_sme2, do_smlal, a, true, gen_helper_sve2_umlsl_zzzw_s)
1540
1541	static bool do_smlal_nx(DisasContext s, arg_azx_n a,
1542	gen_helper_gvec_4 *fn)
1543	{
1544	return do_azz_acc(s, a->n, 2, a->rv, a->off, a->zn, a->zm,
1545	a->idx << 1, 0, false, fn);
1546	}
1547
1548	TRANS_FEAT(SMLAL_nx, aa64_sme2, do_smlal_nx, a, gen_helper_sve2_smlal_idx_s)
1549	TRANS_FEAT(SMLSL_nx, aa64_sme2, do_smlal_nx, a, gen_helper_sve2_smlsl_idx_s)
1550	TRANS_FEAT(UMLAL_nx, aa64_sme2, do_smlal_nx, a, gen_helper_sve2_umlal_idx_s)
1551	TRANS_FEAT(UMLSL_nx, aa64_sme2, do_smlal_nx, a, gen_helper_sve2_umlsl_idx_s)
1552
1553	static bool do_smlall(DisasContext s, arg_azz_n a, bool multi,
1554	gen_helper_gvec_4 *fn)
1555	{
1556	return do_azz_acc(s, a->n, 4, a->rv, a->off, a->zn, a->zm,
1557	0, 0, multi, fn);
1558	}
1559
1560	static void gen_helper_sme2_sumlall_s(TCGv_ptr d, TCGv_ptr n, TCGv_ptr m,
1561	TCGv_ptr a, TCGv_i32 desc)
1562	{
1563	gen_helper_sme2_usmlall_s(d, m, n, a, desc);
1564	}
1565
1566	TRANS_FEAT(SMLALL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_smlall_s)
1567	TRANS_FEAT(SMLSLL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_smlsll_s)
1568	TRANS_FEAT(UMLALL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_umlall_s)
1569	TRANS_FEAT(UMLSLL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_umlsll_s)
1570	TRANS_FEAT(USMLALL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_usmlall_s)
1571	TRANS_FEAT(SUMLALL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_sumlall_s)
1572
1573	TRANS_FEAT(SMLALL_n1_d, aa64_sme2_i16i64, do_smlall, a, false, gen_helper_sme2_smlall_d)
1574	TRANS_FEAT(SMLSLL_n1_d, aa64_sme2_i16i64, do_smlall, a, false, gen_helper_sme2_smlsll_d)
1575	TRANS_FEAT(UMLALL_n1_d, aa64_sme2_i16i64, do_smlall, a, false, gen_helper_sme2_umlall_d)
1576	TRANS_FEAT(UMLSLL_n1_d, aa64_sme2_i16i64, do_smlall, a, false, gen_helper_sme2_umlsll_d)
1577
1578	TRANS_FEAT(SMLALL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_smlall_s)
1579	TRANS_FEAT(SMLSLL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_smlsll_s)
1580	TRANS_FEAT(UMLALL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_umlall_s)
1581	TRANS_FEAT(UMLSLL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_umlsll_s)
1582	TRANS_FEAT(USMLALL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_usmlall_s)
1583
1584	TRANS_FEAT(SMLALL_nn_d, aa64_sme2_i16i64, do_smlall, a, true, gen_helper_sme2_smlall_d)
1585	TRANS_FEAT(SMLSLL_nn_d, aa64_sme2_i16i64, do_smlall, a, true, gen_helper_sme2_smlsll_d)
1586	TRANS_FEAT(UMLALL_nn_d, aa64_sme2_i16i64, do_smlall, a, true, gen_helper_sme2_umlall_d)
1587	TRANS_FEAT(UMLSLL_nn_d, aa64_sme2_i16i64, do_smlall, a, true, gen_helper_sme2_umlsll_d)
1588
1589	static bool do_smlall_nx(DisasContext s, arg_azx_n a,
1590	gen_helper_gvec_4 *fn)
1591	{
1592	return do_azz_acc(s, a->n, 4, a->rv, a->off, a->zn, a->zm,
1593	a->idx << 2, 0, false, fn);
1594	}
1595
1596	TRANS_FEAT(SMLALL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_smlall_idx_s)
1597	TRANS_FEAT(SMLSLL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_smlsll_idx_s)
1598	TRANS_FEAT(UMLALL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_umlall_idx_s)
1599	TRANS_FEAT(UMLSLL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_umlsll_idx_s)
1600	TRANS_FEAT(USMLALL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_usmlall_idx_s)
1601	TRANS_FEAT(SUMLALL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_sumlall_idx_s)
1602
1603	TRANS_FEAT(SMLALL_nx_d, aa64_sme2_i16i64, do_smlall_nx, a, gen_helper_sme2_smlall_idx_d)
1604	TRANS_FEAT(SMLSLL_nx_d, aa64_sme2_i16i64, do_smlall_nx, a, gen_helper_sme2_smlsll_idx_d)
1605	TRANS_FEAT(UMLALL_nx_d, aa64_sme2_i16i64, do_smlall_nx, a, gen_helper_sme2_umlall_idx_d)
1606	TRANS_FEAT(UMLSLL_nx_d, aa64_sme2_i16i64, do_smlall_nx, a, gen_helper_sme2_umlsll_idx_d)
1607
1608	static bool do_zz_fpst(DisasContext s, arg_zz_n a, int data,
1609	ARMFPStatusFlavour type, gen_helper_gvec_2_ptr *fn)
1610	{
1611	if (sme_sm_enabled_check(s)) {
1612	int svl = streaming_vec_reg_size(s);
1613	TCGv_ptr fpst = fpstatus_ptr(type);
1614
1615	for (int i = 0, n = a->n; i < n; ++i) {
1616	tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->zd + i),
1617	vec_full_reg_offset(s, a->zn + i),
1618	fpst, svl, svl, data, fn);
1619	}
1620	}
1621	return true;
1622	}
1623
1624	TRANS_FEAT(BFCVT_hs, aa64_sme2, do_zz_fpst, a, 0,
1625	s->fpcr_ah ? FPST_AH : FPST_A64, gen_helper_sme2_bfcvt_hs)
1626	TRANS_FEAT(BFCVTN, aa64_sme2, do_zz_fpst, a, 0,
1627	s->fpcr_ah ? FPST_AH : FPST_A64, gen_helper_sme2_bfcvtn)
1628	TRANS_FEAT(FCVT_n, aa64_sme2, do_zz_fpst, a, 0,
1629	FPST_A64, gen_helper_sme2_fcvt_n)
1630	TRANS_FEAT(FCVTN, aa64_sme2, do_zz_fpst, a, 0,
1631	FPST_A64, gen_helper_sme2_fcvtn)
1632
1633	TRANS_FEAT(FCVT_w, aa64_sme_f16f16, do_zz_fpst, a, 0,
1634	FPST_A64_F16, gen_helper_sme2_fcvt_w)
1635	TRANS_FEAT(FCVTL, aa64_sme_f16f16, do_zz_fpst, a, 0,
1636	FPST_A64_F16, gen_helper_sme2_fcvtl)
1637
1638	TRANS_FEAT(FCVTZS, aa64_sme2, do_zz_fpst, a, 0,
1639	FPST_A64, gen_helper_gvec_vcvt_rz_fs)
1640	TRANS_FEAT(FCVTZU, aa64_sme2, do_zz_fpst, a, 0,
1641	FPST_A64, gen_helper_gvec_vcvt_rz_fu)
1642
1643	TRANS_FEAT(SCVTF, aa64_sme2, do_zz_fpst, a, 0,
1644	FPST_A64, gen_helper_sme2_scvtf)
1645	TRANS_FEAT(UCVTF, aa64_sme2, do_zz_fpst, a, 0,
1646	FPST_A64, gen_helper_sme2_ucvtf)
1647
1648	TRANS_FEAT(FRINTN, aa64_sme2, do_zz_fpst, a, float_round_nearest_even,
1649	FPST_A64, gen_helper_gvec_vrint_rm_s)
1650	TRANS_FEAT(FRINTP, aa64_sme2, do_zz_fpst, a, float_round_up,
1651	FPST_A64, gen_helper_gvec_vrint_rm_s)
1652	TRANS_FEAT(FRINTM, aa64_sme2, do_zz_fpst, a, float_round_down,
1653	FPST_A64, gen_helper_gvec_vrint_rm_s)
1654	TRANS_FEAT(FRINTA, aa64_sme2, do_zz_fpst, a, float_round_ties_away,
1655	FPST_A64, gen_helper_gvec_vrint_rm_s)
1656
1657	static bool do_zz(DisasContext s, arg_zz_n a, int data,
1658	gen_helper_gvec_2 *fn)
1659	{
1660	if (sme_sm_enabled_check(s)) {
1661	int svl = streaming_vec_reg_size(s);
1662
1663	for (int i = 0, n = a->n; i < n; ++i) {
1664	tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->zd + i),
1665	vec_full_reg_offset(s, a->zn + i),
1666	svl, svl, data, fn);
1667	}
1668	}
1669	return true;
1670	}
1671
1672	TRANS_FEAT(SQCVT_sh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvt_sh)
1673	TRANS_FEAT(UQCVT_sh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvt_sh)
1674	TRANS_FEAT(SQCVTU_sh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtu_sh)
1675
1676	TRANS_FEAT(SQCVT_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvt_sb)
1677	TRANS_FEAT(UQCVT_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvt_sb)
1678	TRANS_FEAT(SQCVTU_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtu_sb)
1679
1680	TRANS_FEAT(SQCVT_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvt_dh)
1681	TRANS_FEAT(UQCVT_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvt_dh)
1682	TRANS_FEAT(SQCVTU_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtu_dh)
1683
1684	TRANS_FEAT(SQCVTN_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtn_sb)
1685	TRANS_FEAT(UQCVTN_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvtn_sb)
1686	TRANS_FEAT(SQCVTUN_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtun_sb)
1687
1688	TRANS_FEAT(SQCVTN_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtn_dh)
1689	TRANS_FEAT(UQCVTN_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvtn_dh)
1690	TRANS_FEAT(SQCVTUN_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtun_dh)
1691
1692	TRANS_FEAT(SUNPK_2bh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk2_bh)
1693	TRANS_FEAT(SUNPK_2hs, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk2_hs)
1694	TRANS_FEAT(SUNPK_2sd, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk2_sd)
1695
1696	TRANS_FEAT(SUNPK_4bh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk4_bh)
1697	TRANS_FEAT(SUNPK_4hs, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk4_hs)
1698	TRANS_FEAT(SUNPK_4sd, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk4_sd)
1699
1700	TRANS_FEAT(UUNPK_2bh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk2_bh)
1701	TRANS_FEAT(UUNPK_2hs, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk2_hs)
1702	TRANS_FEAT(UUNPK_2sd, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk2_sd)
1703
1704	TRANS_FEAT(UUNPK_4bh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk4_bh)
1705	TRANS_FEAT(UUNPK_4hs, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk4_hs)
1706	TRANS_FEAT(UUNPK_4sd, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk4_sd)
1707
1708	static bool do_f8cvt(DisasContext s, arg_zz_n a,
1709	gen_helper_gvec_2_ptr *fn, bool issrc2)
1710	{
1711	if (fpmr_access_check(s) && sme_sm_enabled_check(s)) {
1712	int svl = streaming_vec_reg_size(s);
1713	tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->zd),
1714	vec_full_reg_offset(s, a->zn),
1715	tcg_env, svl, svl,
1716	issrc2 \| (FPST_ZA << 2), fn);
1717	}
1718	return true;
1719	}
1720
1721	TRANS_FEAT(F1CVT, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_fcvt_hb, 0)
1722	TRANS_FEAT(F2CVT, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_fcvt_hb, 1)
1723	TRANS_FEAT(F1CVTL, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_fcvtl_hb, 0)
1724	TRANS_FEAT(F2CVTL, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_fcvtl_hb, 1)
1725
1726	TRANS_FEAT(BF1CVT, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_bfcvt_hb, 0)
1727	TRANS_FEAT(BF2CVT, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_bfcvt_hb, 1)
1728	TRANS_FEAT(BF1CVTL, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_bfcvtl_hb, 0)
1729	TRANS_FEAT(BF2CVTL, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_bfcvtl_hb, 1)
1730
1731	static bool trans_FCVT_bh(DisasContext s, arg_zz_n a)
1732	{
1733	if (!dc_isar_feature(aa64_sme2_f8cvt, s)) {
1734	return false;
1735	}
1736	if (fpmr_access_check(s) && sme_sm_enabled_check(s)) {
1737	int svl = streaming_vec_reg_size(s);
1738	tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->zd),
1739	vec_full_reg_offset(s, a->zn),
1740	vec_full_reg_offset(s, a->zn + 1),
1741	tcg_env, svl, svl,
1742	FPST_ZA << 2, gen_helper_gvec_fcvt_bh);
1743	}
1744	return true;
1745	}
1746
1747	TRANS_FEAT(FCVT_bs, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_fcvt_bs, 0)
1748	TRANS_FEAT(FCVTN_bs, aa64_sme2_f8cvt, do_f8cvt, a, gen_helper_sme2_fcvtn_bs, 0)
1749
1750	static bool do_zipuzp_4(DisasContext s, arg_zz_e a,
1751	gen_helper_gvec_2 * const fn[5])
1752	{
1753	int bytes_per_op = 4 << a->esz;
1754
1755	/* Both MO_64 and MO_128 can fail the size test. */
1756	if (s->max_svl < bytes_per_op) {
1757	unallocated_encoding(s);
1758	} else if (sme_sm_enabled_check(s)) {
1759	int svl = streaming_vec_reg_size(s);
1760	if (svl < bytes_per_op) {
1761	unallocated_encoding(s);
1762	} else {
1763	tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->zd),
1764	vec_full_reg_offset(s, a->zn),
1765	svl, svl, 0, fn[a->esz]);
1766	}
1767	}
1768	return true;
1769	}
1770
1771	static gen_helper_gvec_2 * const zip4_fns[] = {
1772	gen_helper_sme2_zip4_b,
1773	gen_helper_sme2_zip4_h,
1774	gen_helper_sme2_zip4_s,
1775	gen_helper_sme2_zip4_d,
1776	gen_helper_sme2_zip4_q,
1777	};
1778	TRANS_FEAT(ZIP_4, aa64_sme2, do_zipuzp_4, a, zip4_fns)
1779
1780	static gen_helper_gvec_2 * const uzp4_fns[] = {
1781	gen_helper_sme2_uzp4_b,
1782	gen_helper_sme2_uzp4_h,
1783	gen_helper_sme2_uzp4_s,
1784	gen_helper_sme2_uzp4_d,
1785	gen_helper_sme2_uzp4_q,
1786	};
1787	TRANS_FEAT(UZP_4, aa64_sme2, do_zipuzp_4, a, uzp4_fns)
1788
1789	static bool do_zz_rshr(DisasContext s, arg_rshr a, gen_helper_gvec_2 *fn)
1790	{
1791	if (sve_access_check(s)) {
1792	int vl = vec_full_reg_size(s);
1793	tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->zd),
1794	vec_full_reg_offset(s, a->zn),
1795	vl, vl, a->shift, fn);
1796	}
1797	return true;
1798	}
1799
1800	TRANS_FEAT(SQRSHR_sh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshr_sh)
1801	TRANS_FEAT(UQRSHR_sh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshr_sh)
1802	TRANS_FEAT(SQRSHRU_sh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshru_sh)
1803
1804	TRANS_FEAT(SQRSHR_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshr_sb)
1805	TRANS_FEAT(SQRSHR_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshr_dh)
1806	TRANS_FEAT(UQRSHR_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshr_sb)
1807	TRANS_FEAT(UQRSHR_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshr_dh)
1808	TRANS_FEAT(SQRSHRU_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshru_sb)
1809	TRANS_FEAT(SQRSHRU_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshru_dh)
1810
1811	TRANS_FEAT(SQRSHRN_sh, aa64_sme2_or_sve2p1, do_zz_rshr, a, gen_helper_sme2_sqrshrn_sh)
1812	TRANS_FEAT(UQRSHRN_sh, aa64_sme2_or_sve2p1, do_zz_rshr, a, gen_helper_sme2_uqrshrn_sh)
1813	TRANS_FEAT(SQRSHRUN_sh, aa64_sme2_or_sve2p1, do_zz_rshr, a, gen_helper_sme2_sqrshrun_sh)
1814
1815	TRANS_FEAT(SQRSHRN_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshrn_sb)
1816	TRANS_FEAT(SQRSHRN_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshrn_dh)
1817	TRANS_FEAT(UQRSHRN_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshrn_sb)
1818	TRANS_FEAT(UQRSHRN_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshrn_dh)
1819	TRANS_FEAT(SQRSHRUN_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshrun_sb)
1820	TRANS_FEAT(SQRSHRUN_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshrun_dh)
1821
1822	static bool do_zipuzp_2(DisasContext s, arg_zzz_e a,
1823	gen_helper_gvec_3 * const fn[5])
1824	{
1825	int bytes_per_op = 2 << a->esz;
1826
1827	/* MO_128 can fail the size test. */
1828	if (s->max_svl < bytes_per_op) {
1829	unallocated_encoding(s);
1830	} else if (sme_sm_enabled_check(s)) {
1831	int svl = streaming_vec_reg_size(s);
1832	if (svl < bytes_per_op) {
1833	unallocated_encoding(s);
1834	} else {
1835	tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->zd),
1836	vec_full_reg_offset(s, a->zn),
1837	vec_full_reg_offset(s, a->zm),
1838	svl, svl, 0, fn[a->esz]);
1839	}
1840	}
1841	return true;
1842	}
1843
1844	static gen_helper_gvec_3 * const zip2_fns[] = {
1845	gen_helper_sme2_zip2_b,
1846	gen_helper_sme2_zip2_h,
1847	gen_helper_sme2_zip2_s,
1848	gen_helper_sme2_zip2_d,
1849	gen_helper_sme2_zip2_q,
1850	};
1851	TRANS_FEAT(ZIP_2, aa64_sme2, do_zipuzp_2, a, zip2_fns)
1852
1853	static gen_helper_gvec_3 * const uzp2_fns[] = {
1854	gen_helper_sme2_uzp2_b,
1855	gen_helper_sme2_uzp2_h,
1856	gen_helper_sme2_uzp2_s,
1857	gen_helper_sme2_uzp2_d,
1858	gen_helper_sme2_uzp2_q,
1859	};
1860	TRANS_FEAT(UZP_2, aa64_sme2, do_zipuzp_2, a, uzp2_fns)
1861
1862	static bool trans_FCLAMP(DisasContext s, arg_zzz_en a)
1863	{
1864	static gen_helper_gvec_3_ptr * const fn[] = {
1865	gen_helper_sme2_bfclamp,
1866	gen_helper_sme2_fclamp_h,
1867	gen_helper_sme2_fclamp_s,
1868	gen_helper_sme2_fclamp_d,
1869	};
1870	TCGv_ptr fpst;
1871	int vl;
1872
1873	if (!dc_isar_feature(aa64_sme2, s)) {
1874	return false;
1875	}
1876	/* This insn uses MO_8 to encode BFloat16. */
1877	if (a->esz == MO_8 && !dc_isar_feature(aa64_sve_b16b16, s)) {
1878	return false;
1879	}
1880	if (!sme_sm_enabled_check(s)) {
1881	return true;
1882	}
1883
1884	fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
1885	vl = vec_full_reg_size(s);
1886
1887	tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->zd),
1888	vec_full_reg_offset(s, a->zn),
1889	vec_full_reg_offset(s, a->zm),
1890	fpst, vl, vl, a->n, fn[a->esz]);
1891	return true;
1892	}
1893
1894	static bool do_clamp(DisasContext s, arg_zzz_en a,
1895	gen_helper_gvec_3 * const fn[4])
1896	{
1897	int vl;
1898
1899	if (!dc_isar_feature(aa64_sme2, s)) {
1900	return false;
1901	}
1902	if (!sme_sm_enabled_check(s)) {
1903	return true;
1904	}
1905
1906	/*
1907	* Clamp is just a min+max, easily supported by most host
1908	* vector operations -- we already have such an expansion in
1909	* translate-sve.c for a single output.
1910	* TODO: Add support in gvec for multiple simultaneous output,
1911	* and/or copy to temporary upon overlap.
1912	*/
1913	vl = vec_full_reg_size(s);
1914	tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->zd),
1915	vec_full_reg_offset(s, a->zn),
1916	vec_full_reg_offset(s, a->zm),
1917	vl, vl, a->n, fn[a->esz]);
1918	return true;
1919	}
1920
1921	static gen_helper_gvec_3 * const sclamp_fns[] = {
1922	gen_helper_sme2_sclamp_b,
1923	gen_helper_sme2_sclamp_h,
1924	gen_helper_sme2_sclamp_s,
1925	gen_helper_sme2_sclamp_d,
1926	};
1927	TRANS(SCLAMP, do_clamp, a, sclamp_fns)
1928
1929	static gen_helper_gvec_3 * const uclamp_fns[] = {
1930	gen_helper_sme2_uclamp_b,
1931	gen_helper_sme2_uclamp_h,
1932	gen_helper_sme2_uclamp_s,
1933	gen_helper_sme2_uclamp_d,
1934	};
1935	TRANS(UCLAMP, do_clamp, a, uclamp_fns)
1936
1937	static bool trans_SEL(DisasContext s, arg_SEL a)
1938	{
1939	typedef void sme_sel_fn(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32, TCGv_i32);
1940	static sme_sel_fn * const fns[4] = {
1941	gen_helper_sme2_sel_b, gen_helper_sme2_sel_h,
1942	gen_helper_sme2_sel_s, gen_helper_sme2_sel_d
1943	};
1944
1945	if (!dc_isar_feature(aa64_sme2, s)) {
1946	return false;
1947	}
1948	if (sme_sm_enabled_check(s)) {
1949	int svl = streaming_vec_reg_size(s);
1950	uint32_t desc = simd_desc(svl, svl, a->n);
1951	TCGv_ptr t_d = tcg_temp_new_ptr();
1952	TCGv_ptr t_n = tcg_temp_new_ptr();
1953	TCGv_ptr t_m = tcg_temp_new_ptr();
1954	TCGv_i32 png = tcg_temp_new_i32();
1955
1956	tcg_gen_addi_ptr(t_d, tcg_env, vec_full_reg_offset(s, a->zd));
1957	tcg_gen_addi_ptr(t_n, tcg_env, vec_full_reg_offset(s, a->zn));
1958	tcg_gen_addi_ptr(t_m, tcg_env, vec_full_reg_offset(s, a->zm));
1959
1960	tcg_gen_ld16u_i32(png, tcg_env, pred_full_reg_offset(s, a->pg)
1961	^ (HOST_BIG_ENDIAN ? 6 : 0));
1962
1963	fns[a->esz](t_d, t_n, t_m, png, tcg_constant_i32(desc));
1964	}
1965	return true;
1966	}
1967
1968	static bool do_lut(DisasContext s, arg_lut a,
1969	gen_helper_gvec_2_ptr *fn, bool strided)
1970	{
1971	if (sme_sm_enabled_check(s) && sme2_zt0_enabled_check(s)) {
1972	int svl = streaming_vec_reg_size(s);
1973	tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->zd),
1974	vec_full_reg_offset(s, a->zn),
1975	tcg_env, svl, svl, strided \| (a->idx << 1), fn);
1976	}
1977	return true;
1978	}
1979
1980	TRANS_FEAT(LUTI2_c_1b, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_1b, false)
1981	TRANS_FEAT(LUTI2_c_1h, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_1h, false)
1982	TRANS_FEAT(LUTI2_c_1s, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_1s, false)
1983
1984	TRANS_FEAT(LUTI2_c_2b, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_2b, false)
1985	TRANS_FEAT(LUTI2_c_2h, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_2h, false)
1986	TRANS_FEAT(LUTI2_c_2s, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_2s, false)
1987
1988	TRANS_FEAT(LUTI2_c_4b, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_4b, false)
1989	TRANS_FEAT(LUTI2_c_4h, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_4h, false)
1990	TRANS_FEAT(LUTI2_c_4s, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_4s, false)
1991
1992	TRANS_FEAT(LUTI4_c_1b, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_1b, false)
1993	TRANS_FEAT(LUTI4_c_1h, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_1h, false)
1994	TRANS_FEAT(LUTI4_c_1s, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_1s, false)
1995
1996	TRANS_FEAT(LUTI4_c_2b, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_2b, false)
1997	TRANS_FEAT(LUTI4_c_2h, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_2h, false)
1998	TRANS_FEAT(LUTI4_c_2s, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_2s, false)
1999
2000	TRANS_FEAT(LUTI4_c_4h, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_4h, false)
2001	TRANS_FEAT(LUTI4_c_4s, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_4s, false)
2002
2003	TRANS_FEAT(LUTI4_c_4b, aa64_sme_lutv2, do_lut, a,
2004	gen_helper_sme2_luti4_4b, false)
2005
2006	static bool do_lut_s4(DisasContext s, arg_lut a, gen_helper_gvec_2_ptr *fn)
2007	{
2008	return !(a->zd & 0b01100) && do_lut(s, a, fn, true);
2009	}
2010
2011	static bool do_lut_s8(DisasContext s, arg_lut a, gen_helper_gvec_2_ptr *fn)
2012	{
2013	return !(a->zd & 0b01000) && do_lut(s, a, fn, true);
2014	}
2015
2016	TRANS_FEAT(LUTI2_s_2b, aa64_sme2p1, do_lut_s8, a, gen_helper_sme2_luti2_2b)
2017	TRANS_FEAT(LUTI2_s_2h, aa64_sme2p1, do_lut_s8, a, gen_helper_sme2_luti2_2h)
2018
2019	TRANS_FEAT(LUTI2_s_4b, aa64_sme2p1, do_lut_s4, a, gen_helper_sme2_luti2_4b)
2020	TRANS_FEAT(LUTI2_s_4h, aa64_sme2p1, do_lut_s4, a, gen_helper_sme2_luti2_4h)
2021
2022	TRANS_FEAT(LUTI4_s_2b, aa64_sme2p1, do_lut_s8, a, gen_helper_sme2_luti4_2b)
2023	TRANS_FEAT(LUTI4_s_2h, aa64_sme2p1, do_lut_s8, a, gen_helper_sme2_luti4_2h)
2024
2025	TRANS_FEAT(LUTI4_s_4h, aa64_sme2p1, do_lut_s4, a, gen_helper_sme2_luti4_4h)
2026
2027	TRANS_FEAT(LUTI4_s_4b, aa64_sme2p1_lutv2, do_lut_s4, a,
2028	gen_helper_sme2_luti4_4b)
2029
2030	static bool do_mop4_fp(DisasContext s, arg_mop4 a, MemOp esz,
2031	int e_fpst, gen_helper_gvec_3_ptr * const fns[3])
2032	{
2033	int svl = streaming_vec_reg_size(s);
2034	uint32_t desc = simd_desc(svl, svl, (a->m << 1) \| a->n);
2035	int fns_idx = (a->s ? 1 + s->fpcr_ah : 0);
2036	TCGv_ptr za, zn, zm, fpst;
2037
2038	if (!sme_smza_enabled_check(s)) {
2039	return true;
2040	}
2041
2042	za = get_tile(s, esz, a->zad);
2043	zn = vec_full_reg_ptr(s, a->zn);
2044	zm = vec_full_reg_ptr(s, a->zm);
2045	if (e_fpst >= 0) {
2046	fpst = fpstatus_ptr(e_fpst);
2047	} else {
2048	fpst = tcg_env;
2049	}
2050
2051	fns[fns_idx](za, zn, zm, fpst, tcg_constant_i32(desc));
2052	return true;
2053	}
2054
2055	static gen_helper_gvec_3_ptr * const bfmop4_hh[3] = {
2056	gen_helper_sme_bfmop4a_hh,
2057	gen_helper_sme_bfmop4s_hh,
2058	gen_helper_sme_ah_bfmop4s_hh
2059	};
2060	TRANS_FEAT(BFMOP4_hh, aa64_sme_mop4_b16b16,
2061	do_mop4_fp, a, MO_16, FPST_ZA, bfmop4_hh)
2062
2063	static gen_helper_gvec_3_ptr * const fmop4_hh[3] = {
2064	gen_helper_sme_fmop4a_hh,
2065	gen_helper_sme_fmop4s_hh,
2066	gen_helper_sme_ah_fmop4s_hh
2067	};
2068	TRANS_FEAT(FMOP4_hh, aa64_sme_mop4_f16f16,
2069	do_mop4_fp, a, MO_16, FPST_ZA_F16, fmop4_hh)
2070
2071	static gen_helper_gvec_3_ptr * const fmop4_ss[3] = {
2072	gen_helper_sme_fmop4a_ss,
2073	gen_helper_sme_fmop4s_ss,
2074	gen_helper_sme_ah_fmop4s_ss
2075	};
2076	TRANS_FEAT(FMOP4_ss, aa64_sme_mop4, do_mop4_fp, a, MO_32, FPST_ZA, fmop4_ss)
2077
2078	static gen_helper_gvec_3_ptr * const fmop4_dd[3] = {
2079	gen_helper_sme_fmop4a_dd,
2080	gen_helper_sme_fmop4s_dd,
2081	gen_helper_sme_ah_fmop4s_dd
2082	};
2083	TRANS_FEAT(FMOP4_dd, aa64_sme_mop4_f64f64,
2084	do_mop4_fp, a, MO_64, FPST_ZA, fmop4_dd)
2085
2086	static gen_helper_gvec_3_ptr * const bfmop4_sh[3] = {
2087	gen_helper_sme_bfmop4a_sh,
2088	gen_helper_sme_bfmop4s_sh,
2089	gen_helper_sme_ah_bfmop4s_sh
2090	};
2091	TRANS_FEAT(BFMOP4_sh, aa64_sme_mop4, do_mop4_fp, a, MO_32, FPST_ENV, bfmop4_sh)
2092
2093	static gen_helper_gvec_3_ptr * const fmop4_sh[3] = {
2094	gen_helper_sme_fmop4a_sh,
2095	gen_helper_sme_fmop4s_sh,
2096	gen_helper_sme_ah_fmop4s_sh
2097	};
2098	TRANS_FEAT(FMOP4_sh, aa64_sme_mop4, do_mop4_fp, a, MO_32, FPST_ENV, fmop4_sh)
2099
2100	static bool do_mop4_fp8(DisasContext s, arg_mop4 a, MemOp esz,
2101	gen_helper_gvec_3_ptr *fn)
2102	{
2103	if (fpmr_access_check(s) && sme_smza_enabled_check(s)) {
2104	int svl = streaming_vec_reg_size(s);
2105	uint32_t desc = simd_desc(svl, svl, (a->m << 1) \| a->n);
2106	TCGv_ptr za = get_tile(s, esz, a->zad);
2107	TCGv_ptr zn = vec_full_reg_ptr(s, a->zn);
2108	TCGv_ptr zm = vec_full_reg_ptr(s, a->zm);
2109
2110	fn(za, zn, zm, tcg_env, tcg_constant_i32(desc));
2111	}
2112	return true;
2113	}
2114
2115	TRANS_FEAT(FMOP4A_sb, aa64_sme_mop4_f8f32,
2116	do_mop4_fp8, a, MO_32, gen_helper_sme_fmop4a_sb)
2117	TRANS_FEAT(FMOP4A_hb, aa64_sme_mop4_f8f16,
2118	do_mop4_fp8, a, MO_16, gen_helper_sme_fmop4a_hb)
2119
2120	static bool do_mop4_int(DisasContext s, arg_mop4 a, MemOp esz,
2121	gen_helper_gvec_3 *fn)
2122	{
2123	if (sme_smza_enabled_check(s)) {
2124	int svl = streaming_vec_reg_size(s);
2125	uint32_t desc = simd_desc(svl, svl, (a->m << 1) \| a->n);
2126	TCGv_ptr za = get_tile(s, esz, a->zad);
2127	TCGv_ptr zn = vec_full_reg_ptr(s, a->zn);
2128	TCGv_ptr zm = vec_full_reg_ptr(s, a->zm);
2129
2130	fn(za, zn, zm, tcg_constant_i32(desc));
2131	}
2132	return true;
2133	}
2134
2135	TRANS_FEAT(SMOP4_sh, aa64_sme_mop4, do_mop4_int, a, MO_32,
2136	a->s ? gen_helper_sme_smop4s_sh : gen_helper_sme_smop4a_sh)
2137	TRANS_FEAT(UMOP4_sh, aa64_sme_mop4, do_mop4_int, a, MO_32,
2138	a->s ? gen_helper_sme_umop4s_sh : gen_helper_sme_umop4a_sh)
2139
2140	TRANS_FEAT(SMOP4_sb, aa64_sme_mop4, do_mop4_int, a, MO_32,
2141	a->s ? gen_helper_sme_smop4s_sb : gen_helper_sme_smop4a_sb)
2142	TRANS_FEAT(SMOP4_dh, aa64_sme_mop4_i16i64, do_mop4_int, a, MO_64,
2143	a->s ? gen_helper_sme_smop4s_dh : gen_helper_sme_smop4a_dh)
2144
2145	TRANS_FEAT(SUMOP4_sb, aa64_sme_mop4, do_mop4_int, a, MO_32,
2146	a->s ? gen_helper_sme_sumop4s_sb : gen_helper_sme_sumop4a_sb)
2147	TRANS_FEAT(SUMOP4_dh, aa64_sme_mop4_i16i64, do_mop4_int, a, MO_64,
2148	a->s ? gen_helper_sme_sumop4s_dh : gen_helper_sme_sumop4a_dh)
2149
2150	TRANS_FEAT(UMOP4_sb, aa64_sme_mop4, do_mop4_int, a, MO_32,
2151	a->s ? gen_helper_sme_umop4s_sb : gen_helper_sme_umop4a_sb)
2152	TRANS_FEAT(UMOP4_dh, aa64_sme_mop4_i16i64, do_mop4_int, a, MO_64,
2153	a->s ? gen_helper_sme_umop4s_dh : gen_helper_sme_umop4a_dh)
2154
2155	TRANS_FEAT(USMOP4_sb, aa64_sme_mop4, do_mop4_int, a, MO_32,
2156	a->s ? gen_helper_sme_usmop4s_sb : gen_helper_sme_usmop4a_sb)
2157	TRANS_FEAT(USMOP4_dh, aa64_sme_mop4_i16i64, do_mop4_int, a, MO_64,
2158	a->s ? gen_helper_sme_usmop4s_dh : gen_helper_sme_usmop4a_dh)
2159
2160	static bool do_tmop_fp(DisasContext s, arg_tmop a, MemOp esz,
2161	int e_fpst, gen_helper_gvec_4_ptr *fn)
2162	{
2163	if (sme_smza_enabled_check(s)) {
2164	int svl = streaming_vec_reg_size(s);
2165	uint32_t desc = simd_desc(svl, svl, a->idx);
2166	TCGv_ptr za = get_tile(s, esz, a->zad);
2167	TCGv_ptr zn = vec_full_reg_ptr(s, a->zn);
2168	TCGv_ptr zm = vec_full_reg_ptr(s, a->zm);
2169	TCGv_ptr zk = vec_full_reg_ptr(s, a->zm);
2170	TCGv_ptr fpst = (e_fpst >= 0 ? fpstatus_ptr(e_fpst) : tcg_env);
2171
2172	fn(za, zn, zm, zk, fpst, tcg_constant_i32(desc));
2173	}
2174	return true;
2175	}
2176
2177	TRANS_FEAT(BFTMOPA_hh, aa64_sme_tmop_b16b16, do_tmop_fp,
2178	a, MO_16, FPST_ZA, gen_helper_sme_bftmopa_hh)
2179	TRANS_FEAT(FTMOPA_hh, aa64_sme_tmop_f16f16, do_tmop_fp,
2180	a, MO_16, FPST_ZA_F16, gen_helper_sme_ftmopa_hh)
2181	TRANS_FEAT(FTMOPA_ss, aa64_sme_tmop, do_tmop_fp,
2182	a, MO_32, FPST_ZA, gen_helper_sme_ftmopa_ss)
2183
2184	TRANS_FEAT(BFTMOPA_sh, aa64_sme_tmop, do_tmop_fp,
2185	a, MO_32, FPST_ENV, gen_helper_sme_bftmopa_sh)
2186	TRANS_FEAT(FTMOPA_sh, aa64_sme_tmop, do_tmop_fp,
2187	a, MO_32, FPST_ENV, gen_helper_sme_ftmopa_sh)
2188
2189	static bool do_tmop_fp8(DisasContext s, arg_tmop a, MemOp esz,
2190	gen_helper_gvec_4_ptr *fn)
2191	{
2192	if (!fpmr_access_check(s)) {
2193	return true;
2194	}
2195	return do_tmop_fp(s, a, esz, FPST_ENV, fn);
2196	}
2197
2198	TRANS_FEAT(FTMOPA_hb, aa64_sme_tmop_f8f16, do_tmop_fp8,
2199	a, MO_16, gen_helper_sme_ftmopa_hb)
2200	TRANS_FEAT(FTMOPA_sb, aa64_sme_tmop_f8f32, do_tmop_fp8,
2201	a, MO_32, gen_helper_sme_ftmopa_sb)
2202
2203	static bool do_tmop_int(DisasContext s, arg_tmop a, MemOp esz,
2204	gen_helper_gvec_4 *fn)
2205	{
2206	if (sme_smza_enabled_check(s)) {
2207	int svl = streaming_vec_reg_size(s);
2208	uint32_t desc = simd_desc(svl, svl, a->idx);
2209	TCGv_ptr za = get_tile(s, esz, a->zad);
2210	TCGv_ptr zn = vec_full_reg_ptr(s, a->zn);
2211	TCGv_ptr zm = vec_full_reg_ptr(s, a->zm);
2212	TCGv_ptr zk = vec_full_reg_ptr(s, a->zm);
2213
2214	fn(za, zn, zm, zk, tcg_constant_i32(desc));
2215	}
2216	return true;
2217	}
2218
2219	TRANS_FEAT(STMOPA_sh, aa64_sme_tmop, do_tmop_int,
2220	a, MO_32, gen_helper_sme_stmopa_sh)
2221	TRANS_FEAT(UTMOPA_sh, aa64_sme_tmop, do_tmop_int,
2222	a, MO_32, gen_helper_sme_utmopa_sh)
2223
2224	TRANS_FEAT(STMOPA_sb, aa64_sme_tmop, do_tmop_int,
2225	a, MO_32, gen_helper_sme_stmopa_sb)
2226	TRANS_FEAT(SUTMOPA_sb, aa64_sme_tmop, do_tmop_int,
2227	a, MO_32, gen_helper_sme_sutmopa_sb)
2228	TRANS_FEAT(USTMOPA_sb, aa64_sme_tmop, do_tmop_int,
2229	a, MO_32, gen_helper_sme_ustmopa_sb)
2230	TRANS_FEAT(UTMOPA_sb, aa64_sme_tmop, do_tmop_int,
2231	a, MO_32, gen_helper_sme_utmopa_sb)