trans_rvv.c.inc · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

inc 4,093 lines 155 KB

Raw

1	/*
2	*
3	* Copyright (c) 2020 T-Head Semiconductor Co., Ltd. All rights reserved.
4	*
5	* This program is free software; you can redistribute it and/or modify it
6	* under the terms and conditions of the GNU General Public License,
7	* version 2 or later, as published by the Free Software Foundation.
8	*
9	* This program is distributed in the hope it will be useful, but WITHOUT
10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12	* more details.
13	*
14	* You should have received a copy of the GNU General Public License along with
15	* this program. If not, see <http://www.gnu.org/licenses/>.
16	*/
17	#include "tcg/tcg-op-gvec.h"
18	#include "tcg/tcg-gvec-desc.h"
19	#include "internals.h"
20
21	static inline bool is_overlapped(const int8_t astart, int8_t asize,
22	const int8_t bstart, int8_t bsize)
23	{
24	const int8_t aend = astart + asize;
25	const int8_t bend = bstart + bsize;
26
27	return MAX(aend, bend) - MIN(astart, bstart) < asize + bsize;
28	}
29
30	static bool require_rvv(DisasContext *s)
31	{
32	return s->mstatus_vs != EXT_STATUS_DISABLED;
33	}
34
35	static bool require_rvf(DisasContext *s)
36	{
37	if (s->mstatus_fs == EXT_STATUS_DISABLED) {
38	return false;
39	}
40
41	switch (s->sew) {
42	case MO_16:
43	if (s->altfmt) {
44	return s->cfg_ptr->ext_zvfbfa;
45	}
46	return s->cfg_ptr->ext_zvfh;
47	case MO_32:
48	return s->cfg_ptr->ext_zve32f;
49	case MO_64:
50	return s->cfg_ptr->ext_zve64d;
51	default:
52	return false;
53	}
54	}
55
56	static bool require_rvfmin(DisasContext *s)
57	{
58	if (s->mstatus_fs == EXT_STATUS_DISABLED) {
59	return false;
60	}
61
62	switch (s->sew) {
63	case MO_16:
64	if (s->altfmt) {
65	return s->cfg_ptr->ext_zvfbfa;
66	}
67	return s->cfg_ptr->ext_zvfhmin;
68	case MO_32:
69	return s->cfg_ptr->ext_zve32f;
70	default:
71	return false;
72	}
73	}
74
75	static bool require_scale_rvf(DisasContext *s)
76	{
77	if (s->mstatus_fs == EXT_STATUS_DISABLED) {
78	return false;
79	}
80
81	switch (s->sew) {
82	case MO_8:
83	if (s->altfmt) {
84	return s->cfg_ptr->ext_zvfbfa;
85	}
86	return s->cfg_ptr->ext_zvfh;
87	case MO_16:
88	return s->cfg_ptr->ext_zve32f;
89	case MO_32:
90	return s->cfg_ptr->ext_zve64d;
91	default:
92	return false;
93	}
94	}
95
96	static bool require_scale_rvfmin(DisasContext *s)
97	{
98	if (s->mstatus_fs == EXT_STATUS_DISABLED) {
99	return false;
100	}
101
102	switch (s->sew) {
103	case MO_16:
104	return s->cfg_ptr->ext_zve32f;
105	case MO_32:
106	return s->cfg_ptr->ext_zve64d;
107	default:
108	return false;
109	}
110	}
111
112	/*
113	* Source and destination vector register groups cannot overlap source mask
114	* register:
115	*
116	* A vector register cannot be used to provide source operands with more than
117	* one EEW for a single instruction. A mask register source is considered to
118	* have EEW=1 for this constraint. An encoding that would result in the same
119	* vector register being read with two or more different EEWs, including when
120	* the vector register appears at different positions within two or more vector
121	* register groups, is reserved.
122	* (Section 5.2)
123	*
124	* A destination vector register group can overlap a source vector
125	* register group only if one of the following holds:
126	* 1. The destination EEW equals the source EEW.
127	* 2. The destination EEW is smaller than the source EEW and the overlap
128	* is in the lowest-numbered part of the source register group.
129	* 3. The destination EEW is greater than the source EEW, the source EMUL
130	* is at least 1, and the overlap is in the highest-numbered part of
131	* the destination register group.
132	* For the purpose of determining register group overlap constraints, mask
133	* elements have EEW=1.
134	* (Section 5.2)
135	*/
136	static bool require_vm(int vm, int v)
137	{
138	return (vm != 0 \|\| v != 0);
139	}
140
141	static bool require_nf(int vd, int nf, int lmul)
142	{
143	int size = nf << MAX(lmul, 0);
144	return size <= 8 && vd + size <= 32;
145	}
146
147	/*
148	* Vector register should aligned with the passed-in LMUL (EMUL).
149	* If LMUL < 0, i.e. fractional LMUL, any vector register is allowed.
150	*/
151	static bool require_align(const int8_t val, const int8_t lmul)
152	{
153	return lmul <= 0 \|\| extract32(val, 0, lmul) == 0;
154	}
155
156	/*
157	* A destination vector register group can overlap a source vector
158	* register group only if one of the following holds:
159	* 1. The destination EEW equals the source EEW.
160	* 2. The destination EEW is smaller than the source EEW and the overlap
161	* is in the lowest-numbered part of the source register group.
162	* 3. The destination EEW is greater than the source EEW, the source EMUL
163	* is at least 1, and the overlap is in the highest-numbered part of
164	* the destination register group.
165	* (Section 5.2)
166	*
167	* This function returns true if one of the following holds:
168	* * Destination vector register group does not overlap a source vector
169	* register group.
170	* * Rule 3 met.
171	* For rule 1, overlap is allowed so this function doesn't need to be called.
172	* For rule 2, (vd == vs). Caller has to check whether: (vd != vs) before
173	* calling this function.
174	*/
175	static bool require_noover(const int8_t dst, const int8_t dst_lmul,
176	const int8_t src, const int8_t src_lmul)
177	{
178	int8_t dst_size = dst_lmul <= 0 ? 1 : 1 << dst_lmul;
179	int8_t src_size = src_lmul <= 0 ? 1 : 1 << src_lmul;
180
181	/* Destination EEW is greater than the source EEW, check rule 3. */
182	if (dst_size > src_size) {
183	if (dst < src &&
184	src_lmul >= 0 &&
185	is_overlapped(dst, dst_size, src, src_size) &&
186	!is_overlapped(dst, dst_size, src + src_size, src_size)) {
187	return true;
188	}
189	}
190
191	return !is_overlapped(dst, dst_size, src, src_size);
192	}
193
194	static bool do_vsetvl(DisasContext *s, int rd, int rs1, TCGv s2)
195	{
196	TCGv s1, dst;
197
198	if (!require_rvv(s) \|\| !s->cfg_ptr->ext_zve32x) {
199	return false;
200	}
201
202	dst = dest_gpr(s, rd);
203
204	if (rd == 0 && rs1 == 0) {
205	s1 = tcg_temp_new();
206	tcg_gen_ext_i32_tl(s1, cpu_vl);
207	} else if (rs1 == 0) {
208	/* As the mask is at least one bit, RV_VLEN_MAX is >= VLMAX */
209	s1 = tcg_constant_tl(RV_VLEN_MAX);
210	} else {
211	s1 = get_gpr(s, rs1, EXT_ZERO);
212	}
213
214	gen_helper_vsetvl(dst, tcg_env, s1, s2, tcg_constant_tl((int) (rd == 0 && rs1 == 0)));
215	gen_set_gpr(s, rd, dst);
216	finalize_rvv_inst(s);
217
218	gen_update_pc(s, s->cur_insn_len);
219	lookup_and_goto_ptr(s);
220	s->base.is_jmp = DISAS_NORETURN;
221	return true;
222	}
223
224	static bool do_vsetivli(DisasContext *s, int rd, TCGv s1, TCGv s2)
225	{
226	TCGv dst;
227
228	if (!require_rvv(s) \|\| !s->cfg_ptr->ext_zve32x) {
229	return false;
230	}
231
232	dst = dest_gpr(s, rd);
233
234	gen_helper_vsetvl(dst, tcg_env, s1, s2, tcg_constant_tl(0));
235	gen_set_gpr(s, rd, dst);
236	finalize_rvv_inst(s);
237	gen_update_pc(s, s->cur_insn_len);
238	lookup_and_goto_ptr(s);
239	s->base.is_jmp = DISAS_NORETURN;
240
241	return true;
242	}
243
244	static bool trans_vsetvl(DisasContext s, arg_vsetvl a)
245	{
246	TCGv s2 = get_gpr(s, a->rs2, EXT_ZERO);
247	return do_vsetvl(s, a->rd, a->rs1, s2);
248	}
249
250	static bool trans_vsetvli(DisasContext s, arg_vsetvli a)
251	{
252	TCGv s2 = tcg_constant_tl(a->zimm);
253	return do_vsetvl(s, a->rd, a->rs1, s2);
254	}
255
256	static bool trans_vsetivli(DisasContext s, arg_vsetivli a)
257	{
258	TCGv s1 = tcg_constant_tl(a->rs1);
259	TCGv s2 = tcg_constant_tl(a->zimm);
260	return do_vsetivli(s, a->rd, s1, s2);
261	}
262
263	/* vector register offset from env */
264	static uint32_t vreg_ofs(DisasContext *s, int reg)
265	{
266	return offsetof(CPURISCVState, vreg) + reg * s->cfg_ptr->vlenb;
267	}
268
269	/* check functions */
270
271	/*
272	* Vector unit-stride, strided, unit-stride segment, strided segment
273	* store check function.
274	*
275	* Rules to be checked here:
276	* 1. EMUL must within the range: 1/8 <= EMUL <= 8. (Section 7.3)
277	* 2. Destination vector register number is multiples of EMUL.
278	* (Section 3.4.2, 7.3)
279	* 3. The EMUL setting must be such that EMUL * NFIELDS ≤ 8. (Section 7.8)
280	* 4. Vector register numbers accessed by the segment load or store
281	* cannot increment past 31. (Section 7.8)
282	*/
283	static bool vext_check_store(DisasContext *s, int vd, int nf, uint8_t eew)
284	{
285	int8_t emul = eew - s->sew + s->lmul;
286	return (emul >= -3 && emul <= 3) &&
287	require_align(vd, emul) &&
288	require_nf(vd, nf, emul);
289	}
290
291	/*
292	* Vector unit-stride, strided, unit-stride segment, strided segment
293	* load check function.
294	*
295	* Rules to be checked here:
296	* 1. All rules applies to store instructions are applies
297	* to load instructions.
298	* 2. Destination vector register group for a masked vector
299	* instruction cannot overlap the source mask register (v0).
300	* (Section 5.3)
301	*/
302	static bool vext_check_load(DisasContext *s, int vd, int nf, int vm,
303	uint8_t eew)
304	{
305	return vext_check_store(s, vd, nf, eew) && require_vm(vm, vd);
306	}
307
308	/*
309	* Vector indexed, indexed segment store check function.
310	*
311	* Rules to be checked here:
312	* 1. EMUL must within the range: 1/8 <= EMUL <= 8. (Section 7.3)
313	* 2. Index vector register number is multiples of EMUL.
314	* (Section 3.4.2, 7.3)
315	* 3. Destination vector register number is multiples of LMUL.
316	* (Section 3.4.2, 7.3)
317	* 4. The EMUL setting must be such that EMUL * NFIELDS ≤ 8. (Section 7.8)
318	* 5. Vector register numbers accessed by the segment load or store
319	* cannot increment past 31. (Section 7.8)
320	*/
321	static bool vext_check_st_index(DisasContext *s, int vd, int vs2, int nf,
322	uint8_t eew)
323	{
324	int8_t emul = eew - s->sew + s->lmul;
325	bool ret = (emul >= -3 && emul <= 3) &&
326	require_align(vs2, emul) &&
327	require_align(vd, s->lmul) &&
328	require_nf(vd, nf, s->lmul);
329
330	/*
331	* V extension supports all vector load and store instructions,
332	* except V extension does not support EEW=64 for index values
333	* when XLEN=32. (Section 18.3)
334	*/
335	if (get_xl(s) == MXL_RV32) {
336	ret &= (eew != MO_64);
337	}
338
339	return ret;
340	}
341
342	/*
343	* Vector indexed, indexed segment load check function.
344	*
345	* Rules to be checked here:
346	* 1. All rules applies to store instructions are applies
347	* to load instructions.
348	* 2. Destination vector register group for a masked vector
349	* instruction cannot overlap the source mask register (v0).
350	* (Section 5.3)
351	* 3. Destination vector register cannot overlap a source vector
352	* register (vs2) group.
353	* (Section 5.2)
354	* 4. Destination vector register groups cannot overlap
355	* the source vector register (vs2) group for
356	* indexed segment load instructions. (Section 7.8.3)
357	*/
358	static bool vext_check_ld_index(DisasContext *s, int vd, int vs2,
359	int nf, int vm, uint8_t eew)
360	{
361	int8_t seg_vd;
362	int8_t emul = eew - s->sew + s->lmul;
363	bool ret = vext_check_st_index(s, vd, vs2, nf, eew) &&
364	require_vm(vm, vd);
365
366	/* Each segment register group has to follow overlap rules. */
367	for (int i = 0; i < nf; ++i) {
368	seg_vd = vd + (1 << MAX(s->lmul, 0)) * i;
369
370	if (eew > s->sew) {
371	if (seg_vd != vs2) {
372	ret &= require_noover(seg_vd, s->lmul, vs2, emul);
373	}
374	} else if (eew < s->sew) {
375	ret &= require_noover(seg_vd, s->lmul, vs2, emul);
376	}
377
378	/*
379	* Destination vector register groups cannot overlap
380	* the source vector register (vs2) group for
381	* indexed segment load instructions.
382	*/
383	if (nf > 1) {
384	ret &= !is_overlapped(seg_vd, 1 << MAX(s->lmul, 0),
385	vs2, 1 << MAX(emul, 0));
386	}
387	}
388	return ret;
389	}
390
391	/*
392	* Check whether a vector register is used to provide source operands with
393	* more than one EEW for the vector instruction.
394	* Returns true if the instruction has valid encoding
395	* Returns false if encoding violates the mismatched input EEWs constraint
396	*/
397	static bool vext_check_input_eew(DisasContext *s, int vs1, uint8_t eew_vs1,
398	int vs2, uint8_t eew_vs2, int vm)
399	{
400	bool is_valid = true;
401	int8_t emul_vs1 = eew_vs1 - s->sew + s->lmul;
402	int8_t emul_vs2 = eew_vs2 - s->sew + s->lmul;
403
404	/* When vm is 0, vs1 & vs2(EEW!=1) group can't overlap v0 (EEW=1) */
405	if ((vs1 != -1 && !require_vm(vm, vs1)) \|\|
406	(vs2 != -1 && !require_vm(vm, vs2))) {
407	is_valid = false;
408	}
409
410	/* When eew_vs1 != eew_vs2, check whether vs1 and vs2 are overlapped */
411	if ((vs1 != -1 && vs2 != -1) && (eew_vs1 != eew_vs2) &&
412	is_overlapped(vs1, 1 << MAX(emul_vs1, 0),
413	vs2, 1 << MAX(emul_vs2, 0))) {
414	is_valid = false;
415	}
416
417	return is_valid;
418	}
419
420	static bool vext_check_ss(DisasContext *s, int vd, int vs, int vm)
421	{
422	return require_vm(vm, vd) &&
423	require_align(vd, s->lmul) &&
424	require_align(vs, s->lmul) &&
425	vext_check_input_eew(s, vs, s->sew, -1, s->sew, vm);
426	}
427
428	/*
429	* Check function for vector instruction with format:
430	* single-width result and single-width sources (SEW = SEW op SEW)
431	*
432	* Rules to be checked here:
433	* 1. Destination vector register group for a masked vector
434	* instruction cannot overlap the source mask register (v0).
435	* (Section 5.3)
436	* 2. Destination vector register number is multiples of LMUL.
437	* (Section 3.4.2)
438	* 3. Source (vs2, vs1) vector register number are multiples of LMUL.
439	* (Section 3.4.2)
440	*/
441	static bool vext_check_sss(DisasContext *s, int vd, int vs1, int vs2, int vm)
442	{
443	return vext_check_ss(s, vd, vs2, vm) &&
444	vext_check_input_eew(s, vs1, s->sew, vs2, s->sew, vm) &&
445	require_align(vs1, s->lmul);
446	}
447
448	static bool vext_check_ms(DisasContext *s, int vd, int vs)
449	{
450	bool ret = require_align(vs, s->lmul);
451	if (vd != vs) {
452	ret &= require_noover(vd, 0, vs, s->lmul);
453	}
454	return ret;
455	}
456
457	/*
458	* Check function for maskable vector instruction with format:
459	* single-width result and single-width sources (SEW = SEW op SEW)
460	*
461	* Rules to be checked here:
462	* 1. Source (vs2, vs1) vector register number are multiples of LMUL.
463	* (Section 3.4.2)
464	* 2. Destination vector register cannot overlap a source vector
465	* register (vs2, vs1) group.
466	* (Section 5.2)
467	* 3. The destination vector register group for a masked vector
468	* instruction cannot overlap the source mask register (v0),
469	* unless the destination vector register is being written
470	* with a mask value (e.g., comparisons) or the scalar result
471	* of a reduction. (Section 5.3)
472	*/
473	static bool vext_check_mss(DisasContext *s, int vd, int vs1, int vs2)
474	{
475	bool ret = vext_check_ms(s, vd, vs2) &&
476	require_align(vs1, s->lmul);
477	if (vd != vs1) {
478	ret &= require_noover(vd, 0, vs1, s->lmul);
479	}
480	return ret;
481	}
482
483	/*
484	* Common check function for vector widening instructions
485	* of double-width result (2*SEW).
486	*
487	* Rules to be checked here:
488	* 1. The largest vector register group used by an instruction
489	* can not be greater than 8 vector registers (Section 5.2):
490	* => LMUL < 8.
491	* => SEW < 64.
492	* 2. Double-width SEW cannot greater than ELEN.
493	* 3. Destination vector register number is multiples of 2 * LMUL.
494	* (Section 3.4.2)
495	* 4. Destination vector register group for a masked vector
496	* instruction cannot overlap the source mask register (v0).
497	* (Section 5.3)
498	*/
499	static bool vext_wide_check_common(DisasContext *s, int vd, int vm)
500	{
501	return (s->lmul <= 2) &&
502	(s->sew < MO_64) &&
503	((s->sew + 1) <= (s->cfg_ptr->elen >> 4)) &&
504	require_align(vd, s->lmul + 1) &&
505	require_vm(vm, vd);
506	}
507
508	/*
509	* Common check function for vector narrowing instructions
510	* of single-width result (SEW) and double-width source (2*SEW).
511	*
512	* Rules to be checked here:
513	* 1. The largest vector register group used by an instruction
514	* can not be greater than 8 vector registers (Section 5.2):
515	* => LMUL < 8.
516	* => SEW < 64.
517	* 2. Double-width SEW cannot greater than ELEN.
518	* 3. Source vector register number is multiples of 2 * LMUL.
519	* (Section 3.4.2)
520	* 4. Destination vector register number is multiples of LMUL.
521	* (Section 3.4.2)
522	* 5. Destination vector register group for a masked vector
523	* instruction cannot overlap the source mask register (v0).
524	* (Section 5.3)
525	*/
526	static bool vext_narrow_check_common(DisasContext *s, int vd, int vs2,
527	int vm)
528	{
529	return (s->lmul <= 2) &&
530	(s->sew < MO_64) &&
531	((s->sew + 1) <= (s->cfg_ptr->elen >> 4)) &&
532	require_align(vs2, s->lmul + 1) &&
533	require_align(vd, s->lmul) &&
534	require_vm(vm, vd);
535	}
536
537	static bool vext_check_ds(DisasContext *s, int vd, int vs, int vm)
538	{
539	return vext_wide_check_common(s, vd, vm) &&
540	vext_check_input_eew(s, vs, s->sew, -1, 0, vm) &&
541	require_align(vs, s->lmul) &&
542	require_noover(vd, s->lmul + 1, vs, s->lmul);
543	}
544
545	static bool vext_check_dd(DisasContext *s, int vd, int vs, int vm)
546	{
547	return vext_wide_check_common(s, vd, vm) &&
548	vext_check_input_eew(s, vs, s->sew + 1, -1, 0, vm) &&
549	require_align(vs, s->lmul + 1);
550	}
551
552	/*
553	* Check function for vector instruction with format:
554	* double-width result and single-width sources (2*SEW = SEW op SEW)
555	*
556	* Rules to be checked here:
557	* 1. All rules in defined in widen common rules are applied.
558	* 2. Source (vs2, vs1) vector register number are multiples of LMUL.
559	* (Section 3.4.2)
560	* 3. Destination vector register cannot overlap a source vector
561	* register (vs2, vs1) group.
562	* (Section 5.2)
563	*/
564	static bool vext_check_dss(DisasContext *s, int vd, int vs1, int vs2, int vm)
565	{
566	return vext_check_ds(s, vd, vs2, vm) &&
567	vext_check_input_eew(s, vs1, s->sew, vs2, s->sew, vm) &&
568	require_align(vs1, s->lmul) &&
569	require_noover(vd, s->lmul + 1, vs1, s->lmul);
570	}
571
572	/*
573	* Check function for vector instruction with format:
574	* double-width result and double-width source1 and single-width
575	* source2 (2SEW = 2SEW op SEW)
576	*
577	* Rules to be checked here:
578	* 1. All rules in defined in widen common rules are applied.
579	* 2. Source 1 (vs2) vector register number is multiples of 2 * LMUL.
580	* (Section 3.4.2)
581	* 3. Source 2 (vs1) vector register number is multiples of LMUL.
582	* (Section 3.4.2)
583	* 4. Destination vector register cannot overlap a source vector
584	* register (vs1) group.
585	* (Section 5.2)
586	*/
587	static bool vext_check_dds(DisasContext *s, int vd, int vs1, int vs2, int vm)
588	{
589	return vext_check_ds(s, vd, vs1, vm) &&
590	vext_check_input_eew(s, vs1, s->sew, vs2, s->sew + 1, vm) &&
591	require_align(vs2, s->lmul + 1);
592	}
593
594	static bool vext_check_sd(DisasContext *s, int vd, int vs, int vm)
595	{
596	bool ret = vext_narrow_check_common(s, vd, vs, vm) &&
597	vext_check_input_eew(s, vs, s->sew + 1, -1, 0, vm);
598	if (vd != vs) {
599	ret &= require_noover(vd, s->lmul, vs, s->lmul + 1);
600	}
601	return ret;
602	}
603
604	/*
605	* Check function for vector instruction with format:
606	* single-width result and double-width source 1 and single-width
607	* source 2 (SEW = 2*SEW op SEW)
608	*
609	* Rules to be checked here:
610	* 1. All rules in defined in narrow common rules are applied.
611	* 2. Destination vector register cannot overlap a source vector
612	* register (vs2) group.
613	* (Section 5.2)
614	* 3. Source 2 (vs1) vector register number is multiples of LMUL.
615	* (Section 3.4.2)
616	*/
617	static bool vext_check_sds(DisasContext *s, int vd, int vs1, int vs2, int vm)
618	{
619	return vext_check_sd(s, vd, vs2, vm) &&
620	vext_check_input_eew(s, vs1, s->sew, vs2, s->sew + 1, vm) &&
621	require_align(vs1, s->lmul);
622	}
623
624	/*
625	* Check function for vector reduction instructions.
626	*
627	* Rules to be checked here:
628	* 1. Source 1 (vs2) vector register number is multiples of LMUL.
629	* (Section 3.4.2)
630	*/
631	static bool vext_check_reduction(DisasContext *s, int vs2)
632	{
633	return require_align(vs2, s->lmul) && s->vstart_eq_zero;
634	}
635
636	/*
637	* Check function for vector slide instructions.
638	*
639	* Rules to be checked here:
640	* 1. Source 1 (vs2) vector register number is multiples of LMUL.
641	* (Section 3.4.2)
642	* 2. Destination vector register number is multiples of LMUL.
643	* (Section 3.4.2)
644	* 3. Destination vector register group for a masked vector
645	* instruction cannot overlap the source mask register (v0).
646	* (Section 5.3)
647	* 4. The destination vector register group for vslideup, vslide1up,
648	* vfslide1up, cannot overlap the source vector register (vs2) group.
649	* (Section 5.2, 16.3.1, 16.3.3)
650	*/
651	static bool vext_check_slide(DisasContext *s, int vd, int vs2,
652	int vm, bool is_over)
653	{
654	bool ret = require_align(vs2, s->lmul) &&
655	require_align(vd, s->lmul) &&
656	require_vm(vm, vd) &&
657	vext_check_input_eew(s, -1, 0, vs2, s->sew, vm);
658
659	if (is_over) {
660	ret &= (vd != vs2);
661	}
662	return ret;
663	}
664
665	/*
666	* In cpu_get_tb_cpu_state(), set VILL if RVV was not present.
667	* So RVV is also be checked in this function.
668	*/
669	static bool vext_check_isa_ill(DisasContext *s)
670	{
671	return !s->vill;
672	}
673
674	/* common translation macro */
675	#define GEN_VEXT_TRANS(NAME, EEW, ARGTYPE, OP, CHECK) \
676	static bool trans_##NAME(DisasContext s, arg_##ARGTYPE a) \
677	{ \
678	if (CHECK(s, a, EEW)) { \
679	return OP(s, a, EEW); \
680	} \
681	return false; \
682	}
683
684	static uint8_t vext_get_emul(DisasContext *s, uint8_t eew)
685	{
686	int8_t emul = eew - s->sew + s->lmul;
687	return emul < 0 ? 0 : emul;
688	}
689
690	/*
691	*** unit stride load and store
692	*/
693	typedef void gen_helper_ldst_us(TCGv_ptr, TCGv_ptr, TCGv,
694	TCGv_env, TCGv_i32);
695
696	static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data,
697	gen_helper_ldst_us fn, DisasContext s,
698	bool is_store)
699	{
700	TCGv_ptr dest, mask;
701	TCGv base;
702	TCGv_i32 desc;
703
704	dest = tcg_temp_new_ptr();
705	mask = tcg_temp_new_ptr();
706	base = get_gpr(s, rs1, EXT_NONE);
707
708	/*
709	* As simd_desc supports at most 2048 bytes, and in this implementation,
710	* the max vector group length is 4096 bytes. So split it into two parts.
711	*
712	* The first part is vlen in bytes (vlenb), encoded in maxsz of simd_desc.
713	* The second part is lmul, encoded in data of simd_desc.
714	*/
715	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
716	s->cfg_ptr->vlenb, data));
717
718	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
719	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
720
721	/*
722	* According to the specification
723	*
724	* Additionally, if the Ztso extension is implemented, then vector memory
725	* instructions in the V extension and Zve family of extensions follow
726	* RVTSO at the instruction level. The Ztso extension does not
727	* strengthen the ordering of intra-instruction element accesses.
728	*
729	* as a result neither ordered nor unordered accesses from the V
730	* instructions need ordering within the loop but we do still need barriers
731	* around the loop.
732	*/
733	if (is_store && s->ztso) {
734	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_STRL);
735	}
736
737	mark_vs_dirty(s);
738
739	fn(dest, mask, base, tcg_env, desc);
740
741	if (!is_store && s->ztso) {
742	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_LDAQ);
743	}
744
745	finalize_rvv_inst(s);
746	return true;
747	}
748
749	static bool ld_us_op(DisasContext s, arg_r2nfvm a, uint8_t eew)
750	{
751	uint32_t data = 0;
752	gen_helper_ldst_us *fn;
753	static gen_helper_ldst_us * const fns[2][4] = {
754	/* masked unit stride load */
755	{ gen_helper_vle8_v_mask, gen_helper_vle16_v_mask,
756	gen_helper_vle32_v_mask, gen_helper_vle64_v_mask },
757	/* unmasked unit stride load */
758	{ gen_helper_vle8_v, gen_helper_vle16_v,
759	gen_helper_vle32_v, gen_helper_vle64_v }
760	};
761
762	fn = fns[a->vm][eew];
763	if (fn == NULL) {
764	return false;
765	}
766
767	/*
768	* Vector load/store instructions have the EEW encoded
769	* directly in the instructions. The maximum vector size is
770	* calculated with EMUL rather than LMUL.
771	*/
772	uint8_t emul = vext_get_emul(s, eew);
773	data = FIELD_DP32(data, VDATA, VM, a->vm);
774	data = FIELD_DP32(data, VDATA, LMUL, emul);
775	data = FIELD_DP32(data, VDATA, NF, a->nf);
776	data = FIELD_DP32(data, VDATA, VTA, s->vta);
777	data = FIELD_DP32(data, VDATA, VMA, s->vma);
778	return ldst_us_trans(a->rd, a->rs1, data, fn, s, false);
779	}
780
781	static bool ld_us_check(DisasContext s, arg_r2nfvm a, uint8_t eew)
782	{
783	return require_rvv(s) &&
784	vext_check_isa_ill(s) &&
785	vext_check_load(s, a->rd, a->nf, a->vm, eew);
786	}
787
788	GEN_VEXT_TRANS(vle8_v, MO_8, r2nfvm, ld_us_op, ld_us_check)
789	GEN_VEXT_TRANS(vle16_v, MO_16, r2nfvm, ld_us_op, ld_us_check)
790	GEN_VEXT_TRANS(vle32_v, MO_32, r2nfvm, ld_us_op, ld_us_check)
791	GEN_VEXT_TRANS(vle64_v, MO_64, r2nfvm, ld_us_op, ld_us_check)
792
793	static bool st_us_op(DisasContext s, arg_r2nfvm a, uint8_t eew)
794	{
795	uint32_t data = 0;
796	gen_helper_ldst_us *fn;
797	static gen_helper_ldst_us * const fns[2][4] = {
798	/* masked unit stride store */
799	{ gen_helper_vse8_v_mask, gen_helper_vse16_v_mask,
800	gen_helper_vse32_v_mask, gen_helper_vse64_v_mask },
801	/* unmasked unit stride store */
802	{ gen_helper_vse8_v, gen_helper_vse16_v,
803	gen_helper_vse32_v, gen_helper_vse64_v }
804	};
805
806	fn = fns[a->vm][eew];
807	if (fn == NULL) {
808	return false;
809	}
810
811	uint8_t emul = vext_get_emul(s, eew);
812	data = FIELD_DP32(data, VDATA, VM, a->vm);
813	data = FIELD_DP32(data, VDATA, LMUL, emul);
814	data = FIELD_DP32(data, VDATA, NF, a->nf);
815	return ldst_us_trans(a->rd, a->rs1, data, fn, s, true);
816	}
817
818	static bool st_us_check(DisasContext s, arg_r2nfvm a, uint8_t eew)
819	{
820	return require_rvv(s) &&
821	vext_check_isa_ill(s) &&
822	vext_check_store(s, a->rd, a->nf, eew);
823	}
824
825	GEN_VEXT_TRANS(vse8_v, MO_8, r2nfvm, st_us_op, st_us_check)
826	GEN_VEXT_TRANS(vse16_v, MO_16, r2nfvm, st_us_op, st_us_check)
827	GEN_VEXT_TRANS(vse32_v, MO_32, r2nfvm, st_us_op, st_us_check)
828	GEN_VEXT_TRANS(vse64_v, MO_64, r2nfvm, st_us_op, st_us_check)
829
830	/*
831	*** unit stride mask load and store
832	*/
833	static bool ld_us_mask_op(DisasContext s, arg_vlm_v a, uint8_t eew)
834	{
835	uint32_t data = 0;
836	gen_helper_ldst_us *fn = gen_helper_vlm_v;
837
838	/* EMUL = 1, NFIELDS = 1 */
839	data = FIELD_DP32(data, VDATA, LMUL, 0);
840	data = FIELD_DP32(data, VDATA, NF, 1);
841	/* Mask destination register are always tail-agnostic */
842	data = FIELD_DP32(data, VDATA, VTA, s->cfg_vta_all_1s);
843	data = FIELD_DP32(data, VDATA, VMA, s->vma);
844	data = FIELD_DP32(data, VDATA, VM, 1);
845	return ldst_us_trans(a->rd, a->rs1, data, fn, s, false);
846	}
847
848	static bool ld_us_mask_check(DisasContext s, arg_vlm_v a, uint8_t eew)
849	{
850	/* EMUL = 1, NFIELDS = 1 */
851	return require_rvv(s) && vext_check_isa_ill(s);
852	}
853
854	static bool st_us_mask_op(DisasContext s, arg_vsm_v a, uint8_t eew)
855	{
856	uint32_t data = 0;
857	gen_helper_ldst_us *fn = gen_helper_vsm_v;
858
859	/* EMUL = 1, NFIELDS = 1 */
860	data = FIELD_DP32(data, VDATA, LMUL, 0);
861	data = FIELD_DP32(data, VDATA, NF, 1);
862	data = FIELD_DP32(data, VDATA, VM, 1);
863	return ldst_us_trans(a->rd, a->rs1, data, fn, s, true);
864	}
865
866	static bool st_us_mask_check(DisasContext s, arg_vsm_v a, uint8_t eew)
867	{
868	/* EMUL = 1, NFIELDS = 1 */
869	return require_rvv(s) && vext_check_isa_ill(s);
870	}
871
872	GEN_VEXT_TRANS(vlm_v, MO_8, vlm_v, ld_us_mask_op, ld_us_mask_check)
873	GEN_VEXT_TRANS(vsm_v, MO_8, vsm_v, st_us_mask_op, st_us_mask_check)
874
875	/*
876	*** stride load and store
877	*/
878	typedef void gen_helper_ldst_stride(TCGv_ptr, TCGv_ptr, TCGv,
879	TCGv, TCGv_env, TCGv_i32);
880
881	static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
882	uint32_t data, gen_helper_ldst_stride *fn,
883	DisasContext *s)
884	{
885	TCGv_ptr dest, mask;
886	TCGv base, stride;
887	TCGv_i32 desc;
888
889	dest = tcg_temp_new_ptr();
890	mask = tcg_temp_new_ptr();
891	base = get_gpr(s, rs1, EXT_NONE);
892	stride = get_gpr(s, rs2, EXT_NONE);
893	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
894	s->cfg_ptr->vlenb, data));
895
896	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
897	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
898
899	mark_vs_dirty(s);
900
901	fn(dest, mask, base, stride, tcg_env, desc);
902
903	finalize_rvv_inst(s);
904	return true;
905	}
906
907	static bool ld_stride_op(DisasContext s, arg_rnfvm a, uint8_t eew)
908	{
909	uint32_t data = 0;
910	gen_helper_ldst_stride *fn;
911	static gen_helper_ldst_stride * const fns[4] = {
912	gen_helper_vlse8_v, gen_helper_vlse16_v,
913	gen_helper_vlse32_v, gen_helper_vlse64_v
914	};
915
916	fn = fns[eew];
917	if (fn == NULL) {
918	return false;
919	}
920
921	uint8_t emul = vext_get_emul(s, eew);
922	data = FIELD_DP32(data, VDATA, VM, a->vm);
923	data = FIELD_DP32(data, VDATA, LMUL, emul);
924	data = FIELD_DP32(data, VDATA, NF, a->nf);
925	data = FIELD_DP32(data, VDATA, VTA, s->vta);
926	data = FIELD_DP32(data, VDATA, VMA, s->vma);
927	return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s);
928	}
929
930	static bool ld_stride_check(DisasContext s, arg_rnfvm a, uint8_t eew)
931	{
932	return require_rvv(s) &&
933	vext_check_isa_ill(s) &&
934	vext_check_load(s, a->rd, a->nf, a->vm, eew);
935	}
936
937	GEN_VEXT_TRANS(vlse8_v, MO_8, rnfvm, ld_stride_op, ld_stride_check)
938	GEN_VEXT_TRANS(vlse16_v, MO_16, rnfvm, ld_stride_op, ld_stride_check)
939	GEN_VEXT_TRANS(vlse32_v, MO_32, rnfvm, ld_stride_op, ld_stride_check)
940	GEN_VEXT_TRANS(vlse64_v, MO_64, rnfvm, ld_stride_op, ld_stride_check)
941
942	static bool st_stride_op(DisasContext s, arg_rnfvm a, uint8_t eew)
943	{
944	uint32_t data = 0;
945	gen_helper_ldst_stride *fn;
946	static gen_helper_ldst_stride * const fns[4] = {
947	/* masked stride store */
948	gen_helper_vsse8_v, gen_helper_vsse16_v,
949	gen_helper_vsse32_v, gen_helper_vsse64_v
950	};
951
952	uint8_t emul = vext_get_emul(s, eew);
953	data = FIELD_DP32(data, VDATA, VM, a->vm);
954	data = FIELD_DP32(data, VDATA, LMUL, emul);
955	data = FIELD_DP32(data, VDATA, NF, a->nf);
956	fn = fns[eew];
957	if (fn == NULL) {
958	return false;
959	}
960
961	return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s);
962	}
963
964	static bool st_stride_check(DisasContext s, arg_rnfvm a, uint8_t eew)
965	{
966	return require_rvv(s) &&
967	vext_check_isa_ill(s) &&
968	vext_check_store(s, a->rd, a->nf, eew);
969	}
970
971	GEN_VEXT_TRANS(vsse8_v, MO_8, rnfvm, st_stride_op, st_stride_check)
972	GEN_VEXT_TRANS(vsse16_v, MO_16, rnfvm, st_stride_op, st_stride_check)
973	GEN_VEXT_TRANS(vsse32_v, MO_32, rnfvm, st_stride_op, st_stride_check)
974	GEN_VEXT_TRANS(vsse64_v, MO_64, rnfvm, st_stride_op, st_stride_check)
975
976	/*
977	*** index load and store
978	*/
979	typedef void gen_helper_ldst_index(TCGv_ptr, TCGv_ptr, TCGv,
980	TCGv_ptr, TCGv_env, TCGv_i32);
981
982	static bool ldst_index_trans(uint32_t vd, uint32_t rs1, uint32_t vs2,
983	uint32_t data, gen_helper_ldst_index *fn,
984	DisasContext *s)
985	{
986	TCGv_ptr dest, mask, index;
987	TCGv base;
988	TCGv_i32 desc;
989
990	dest = tcg_temp_new_ptr();
991	mask = tcg_temp_new_ptr();
992	index = tcg_temp_new_ptr();
993	base = get_gpr(s, rs1, EXT_NONE);
994	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
995	s->cfg_ptr->vlenb, data));
996
997	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
998	tcg_gen_addi_ptr(index, tcg_env, vreg_ofs(s, vs2));
999	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
1000
1001	mark_vs_dirty(s);
1002
1003	fn(dest, mask, base, index, tcg_env, desc);
1004
1005	finalize_rvv_inst(s);
1006	return true;
1007	}
1008
1009	static bool ld_index_op(DisasContext s, arg_rnfvm a, uint8_t eew)
1010	{
1011	uint32_t data = 0;
1012	gen_helper_ldst_index *fn;
1013	static gen_helper_ldst_index * const fns[4][4] = {
1014	/*
1015	* offset vector register group EEW = 8,
1016	* data vector register group EEW = SEW
1017	*/
1018	{ gen_helper_vlxei8_8_v, gen_helper_vlxei8_16_v,
1019	gen_helper_vlxei8_32_v, gen_helper_vlxei8_64_v },
1020	/*
1021	* offset vector register group EEW = 16,
1022	* data vector register group EEW = SEW
1023	*/
1024	{ gen_helper_vlxei16_8_v, gen_helper_vlxei16_16_v,
1025	gen_helper_vlxei16_32_v, gen_helper_vlxei16_64_v },
1026	/*
1027	* offset vector register group EEW = 32,
1028	* data vector register group EEW = SEW
1029	*/
1030	{ gen_helper_vlxei32_8_v, gen_helper_vlxei32_16_v,
1031	gen_helper_vlxei32_32_v, gen_helper_vlxei32_64_v },
1032	/*
1033	* offset vector register group EEW = 64,
1034	* data vector register group EEW = SEW
1035	*/
1036	{ gen_helper_vlxei64_8_v, gen_helper_vlxei64_16_v,
1037	gen_helper_vlxei64_32_v, gen_helper_vlxei64_64_v }
1038	};
1039
1040	fn = fns[eew][s->sew];
1041
1042	uint8_t emul = vext_get_emul(s, s->sew);
1043	data = FIELD_DP32(data, VDATA, VM, a->vm);
1044	data = FIELD_DP32(data, VDATA, LMUL, emul);
1045	data = FIELD_DP32(data, VDATA, NF, a->nf);
1046	data = FIELD_DP32(data, VDATA, VTA, s->vta);
1047	data = FIELD_DP32(data, VDATA, VMA, s->vma);
1048	return ldst_index_trans(a->rd, a->rs1, a->rs2, data, fn, s);
1049	}
1050
1051	static bool ld_index_check(DisasContext s, arg_rnfvm a, uint8_t eew)
1052	{
1053	return require_rvv(s) &&
1054	vext_check_isa_ill(s) &&
1055	vext_check_ld_index(s, a->rd, a->rs2, a->nf, a->vm, eew) &&
1056	vext_check_input_eew(s, -1, 0, a->rs2, eew, a->vm);
1057	}
1058
1059	GEN_VEXT_TRANS(vlxei8_v, MO_8, rnfvm, ld_index_op, ld_index_check)
1060	GEN_VEXT_TRANS(vlxei16_v, MO_16, rnfvm, ld_index_op, ld_index_check)
1061	GEN_VEXT_TRANS(vlxei32_v, MO_32, rnfvm, ld_index_op, ld_index_check)
1062	GEN_VEXT_TRANS(vlxei64_v, MO_64, rnfvm, ld_index_op, ld_index_check)
1063
1064	static bool st_index_op(DisasContext s, arg_rnfvm a, uint8_t eew)
1065	{
1066	uint32_t data = 0;
1067	gen_helper_ldst_index *fn;
1068	static gen_helper_ldst_index * const fns[4][4] = {
1069	/*
1070	* offset vector register group EEW = 8,
1071	* data vector register group EEW = SEW
1072	*/
1073	{ gen_helper_vsxei8_8_v, gen_helper_vsxei8_16_v,
1074	gen_helper_vsxei8_32_v, gen_helper_vsxei8_64_v },
1075	/*
1076	* offset vector register group EEW = 16,
1077	* data vector register group EEW = SEW
1078	*/
1079	{ gen_helper_vsxei16_8_v, gen_helper_vsxei16_16_v,
1080	gen_helper_vsxei16_32_v, gen_helper_vsxei16_64_v },
1081	/*
1082	* offset vector register group EEW = 32,
1083	* data vector register group EEW = SEW
1084	*/
1085	{ gen_helper_vsxei32_8_v, gen_helper_vsxei32_16_v,
1086	gen_helper_vsxei32_32_v, gen_helper_vsxei32_64_v },
1087	/*
1088	* offset vector register group EEW = 64,
1089	* data vector register group EEW = SEW
1090	*/
1091	{ gen_helper_vsxei64_8_v, gen_helper_vsxei64_16_v,
1092	gen_helper_vsxei64_32_v, gen_helper_vsxei64_64_v }
1093	};
1094
1095	fn = fns[eew][s->sew];
1096
1097	uint8_t emul = vext_get_emul(s, s->sew);
1098	data = FIELD_DP32(data, VDATA, VM, a->vm);
1099	data = FIELD_DP32(data, VDATA, LMUL, emul);
1100	data = FIELD_DP32(data, VDATA, NF, a->nf);
1101	return ldst_index_trans(a->rd, a->rs1, a->rs2, data, fn, s);
1102	}
1103
1104	static bool st_index_check(DisasContext s, arg_rnfvm a, uint8_t eew)
1105	{
1106	return require_rvv(s) &&
1107	vext_check_isa_ill(s) &&
1108	vext_check_st_index(s, a->rd, a->rs2, a->nf, eew) &&
1109	vext_check_input_eew(s, a->rd, s->sew, a->rs2, eew, a->vm);
1110	}
1111
1112	GEN_VEXT_TRANS(vsxei8_v, MO_8, rnfvm, st_index_op, st_index_check)
1113	GEN_VEXT_TRANS(vsxei16_v, MO_16, rnfvm, st_index_op, st_index_check)
1114	GEN_VEXT_TRANS(vsxei32_v, MO_32, rnfvm, st_index_op, st_index_check)
1115	GEN_VEXT_TRANS(vsxei64_v, MO_64, rnfvm, st_index_op, st_index_check)
1116
1117	/*
1118	*** unit stride fault-only-first load
1119	*/
1120	static bool ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data,
1121	gen_helper_ldst_us fn, DisasContext s)
1122	{
1123	TCGv_ptr dest, mask;
1124	TCGv base;
1125	TCGv_i32 desc;
1126
1127	dest = tcg_temp_new_ptr();
1128	mask = tcg_temp_new_ptr();
1129	base = get_gpr(s, rs1, EXT_NONE);
1130	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
1131	s->cfg_ptr->vlenb, data));
1132
1133	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
1134	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
1135
1136	fn(dest, mask, base, tcg_env, desc);
1137
1138	finalize_rvv_inst(s);
1139
1140	/* vector unit-stride fault-only-first load may modify vl CSR */
1141	gen_update_pc(s, s->cur_insn_len);
1142	lookup_and_goto_ptr(s);
1143	s->base.is_jmp = DISAS_NORETURN;
1144
1145	return true;
1146	}
1147
1148	static bool ldff_op(DisasContext s, arg_r2nfvm a, uint8_t eew)
1149	{
1150	uint32_t data = 0;
1151	gen_helper_ldst_us *fn;
1152	static gen_helper_ldst_us * const fns[4] = {
1153	gen_helper_vle8ff_v, gen_helper_vle16ff_v,
1154	gen_helper_vle32ff_v, gen_helper_vle64ff_v
1155	};
1156
1157	fn = fns[eew];
1158	if (fn == NULL) {
1159	return false;
1160	}
1161
1162	uint8_t emul = vext_get_emul(s, eew);
1163	data = FIELD_DP32(data, VDATA, VM, a->vm);
1164	data = FIELD_DP32(data, VDATA, LMUL, emul);
1165	data = FIELD_DP32(data, VDATA, NF, a->nf);
1166	data = FIELD_DP32(data, VDATA, VTA, s->vta);
1167	data = FIELD_DP32(data, VDATA, VMA, s->vma);
1168	return ldff_trans(a->rd, a->rs1, data, fn, s);
1169	}
1170
1171	GEN_VEXT_TRANS(vle8ff_v, MO_8, r2nfvm, ldff_op, ld_us_check)
1172	GEN_VEXT_TRANS(vle16ff_v, MO_16, r2nfvm, ldff_op, ld_us_check)
1173	GEN_VEXT_TRANS(vle32ff_v, MO_32, r2nfvm, ldff_op, ld_us_check)
1174	GEN_VEXT_TRANS(vle64ff_v, MO_64, r2nfvm, ldff_op, ld_us_check)
1175
1176	/*
1177	* load and store whole register instructions
1178	*/
1179	typedef void gen_helper_ldst_whole(TCGv_ptr, TCGv, TCGv_env, TCGv_i32);
1180
1181	static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf,
1182	uint32_t log2_esz, gen_helper_ldst_whole *fn,
1183	DisasContext *s, bool is_load)
1184	{
1185	mark_vs_dirty(s);
1186
1187	/*
1188	* Load/store multiple bytes per iteration.
1189	* When possible do this atomically.
1190	* Update vstart with the number of processed elements.
1191	* Use the helper function if either:
1192	* - vstart is not 0.
1193	*/
1194	bool use_helper_fn = !s->vstart_eq_zero;
1195
1196	if (!use_helper_fn) {
1197	uint32_t size = s->cfg_ptr->vlenb * nf;
1198	TCGv_i64 t8 = tcg_temp_new_i64();
1199	MemOp atomicity = MO_ATOM_NONE;
1200	MemOp alignment = MO_UNALN;
1201
1202	/*
1203	* If Zicclsm is disabled, require alignment based on element size.
1204	* Use MO_ALIGN_* based on log2_esz (0 = MO_UNALN, 1 = MO_ALIGN_2, etc).
1205	*/
1206	if (!s->cfg_ptr->ext_zicclsm) {
1207	alignment = log2_esz << MO_ASHIFT;
1208	}
1209
1210	if (log2_esz == 0) {
1211	atomicity = MO_ATOM_NONE;
1212	} else {
1213	atomicity = MO_ATOM_IFALIGN_PAIR;
1214	}
1215
1216	for (int i = 0; i < size; i += 8) {
1217	TCGv addr = get_address(s, rs1, i);
1218	if (is_load) {
1219	tcg_gen_qemu_ld_i64(t8, addr, s->mem_idx,
1220	MO_LEUQ \| atomicity \| alignment);
1221	tcg_gen_st_i64(t8, tcg_env, vreg_ofs(s, vd) + i);
1222	} else {
1223	tcg_gen_ld_i64(t8, tcg_env, vreg_ofs(s, vd) + i);
1224	tcg_gen_qemu_st_i64(t8, addr, s->mem_idx,
1225	MO_LEUQ \| atomicity \| alignment);
1226	}
1227	if (i == size - 8) {
1228	tcg_gen_movi_i32(cpu_vstart, 0);
1229	} else {
1230	tcg_gen_addi_i32(cpu_vstart, cpu_vstart, 8 >> log2_esz);
1231	}
1232	}
1233	} else {
1234	TCGv_ptr dest;
1235	TCGv base;
1236	TCGv_i32 desc;
1237	uint32_t data = FIELD_DP32(0, VDATA, NF, nf);
1238	data = FIELD_DP32(data, VDATA, VM, 1);
1239	dest = tcg_temp_new_ptr();
1240	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
1241	s->cfg_ptr->vlenb, data));
1242	base = get_gpr(s, rs1, EXT_NONE);
1243	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
1244	fn(dest, base, tcg_env, desc);
1245	}
1246
1247	finalize_rvv_inst(s);
1248	return true;
1249	}
1250
1251	/*
1252	* load and store whole register instructions ignore vtype and vl setting.
1253	* Thus, we don't need to check vill bit. (Section 7.9)
1254	*/
1255	#define GEN_LDST_WHOLE_TRANS(NAME, ETYPE, ARG_NF, IS_LOAD) \
1256	static bool trans_##NAME(DisasContext s, arg_##NAME a) \
1257	{ \
1258	if (require_rvv(s) && \
1259	QEMU_IS_ALIGNED(a->rd, ARG_NF)) { \
1260	return ldst_whole_trans(a->rd, a->rs1, ARG_NF, ctzl(sizeof(ETYPE)), \
1261	gen_helper_##NAME, s, IS_LOAD); \
1262	} \
1263	return false; \
1264	}
1265
1266	GEN_LDST_WHOLE_TRANS(vl1re8_v, int8_t, 1, true)
1267	GEN_LDST_WHOLE_TRANS(vl1re16_v, int16_t, 1, true)
1268	GEN_LDST_WHOLE_TRANS(vl1re32_v, int32_t, 1, true)
1269	GEN_LDST_WHOLE_TRANS(vl1re64_v, int64_t, 1, true)
1270	GEN_LDST_WHOLE_TRANS(vl2re8_v, int8_t, 2, true)
1271	GEN_LDST_WHOLE_TRANS(vl2re16_v, int16_t, 2, true)
1272	GEN_LDST_WHOLE_TRANS(vl2re32_v, int32_t, 2, true)
1273	GEN_LDST_WHOLE_TRANS(vl2re64_v, int64_t, 2, true)
1274	GEN_LDST_WHOLE_TRANS(vl4re8_v, int8_t, 4, true)
1275	GEN_LDST_WHOLE_TRANS(vl4re16_v, int16_t, 4, true)
1276	GEN_LDST_WHOLE_TRANS(vl4re32_v, int32_t, 4, true)
1277	GEN_LDST_WHOLE_TRANS(vl4re64_v, int64_t, 4, true)
1278	GEN_LDST_WHOLE_TRANS(vl8re8_v, int8_t, 8, true)
1279	GEN_LDST_WHOLE_TRANS(vl8re16_v, int16_t, 8, true)
1280	GEN_LDST_WHOLE_TRANS(vl8re32_v, int32_t, 8, true)
1281	GEN_LDST_WHOLE_TRANS(vl8re64_v, int64_t, 8, true)
1282
1283	/*
1284	* The vector whole register store instructions are encoded similar to
1285	* unmasked unit-stride store of elements with EEW=8.
1286	*/
1287	GEN_LDST_WHOLE_TRANS(vs1r_v, int8_t, 1, false)
1288	GEN_LDST_WHOLE_TRANS(vs2r_v, int8_t, 2, false)
1289	GEN_LDST_WHOLE_TRANS(vs4r_v, int8_t, 4, false)
1290	GEN_LDST_WHOLE_TRANS(vs8r_v, int8_t, 8, false)
1291
1292	/*
1293	*** Vector Integer Arithmetic Instructions
1294	*/
1295
1296	/*
1297	* MAXSZ returns the maximum vector size can be operated in bytes,
1298	* which is used in GVEC IR when vl_eq_vlmax flag is set to true
1299	* to accelerate vector operation.
1300	*/
1301	static inline uint32_t MAXSZ(DisasContext *s)
1302	{
1303	int max_sz = s->cfg_ptr->vlenb * 8;
1304	return max_sz >> (3 - s->lmul);
1305	}
1306
1307	static bool opivv_check(DisasContext s, arg_rmrr a)
1308	{
1309	return require_rvv(s) &&
1310	vext_check_isa_ill(s) &&
1311	vext_check_sss(s, a->rd, a->rs1, a->rs2, a->vm);
1312	}
1313
1314	typedef void GVecGen3Fn(unsigned, uint32_t, uint32_t,
1315	uint32_t, uint32_t, uint32_t);
1316
1317	static inline bool
1318	do_opivv_gvec(DisasContext s, arg_rmrr a, GVecGen3Fn *gvec_fn,
1319	gen_helper_gvec_4_ptr *fn)
1320	{
1321	if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
1322	gvec_fn(s->sew, vreg_ofs(s, a->rd),
1323	vreg_ofs(s, a->rs2), vreg_ofs(s, a->rs1),
1324	MAXSZ(s), MAXSZ(s));
1325	} else {
1326	uint32_t data = 0;
1327
1328	data = FIELD_DP32(data, VDATA, VM, a->vm);
1329	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1330	data = FIELD_DP32(data, VDATA, VTA, s->vta);
1331	data = FIELD_DP32(data, VDATA, VMA, s->vma);
1332	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
1333	vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2),
1334	tcg_env, s->cfg_ptr->vlenb,
1335	s->cfg_ptr->vlenb, data, fn);
1336	}
1337	finalize_rvv_inst(s);
1338	return true;
1339	}
1340
1341	/* OPIVV with GVEC IR */
1342	#define GEN_OPIVV_GVEC_TRANS(NAME, SUF) \
1343	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1344	{ \
1345	static gen_helper_gvec_4_ptr * const fns[4] = { \
1346	gen_helper_##NAME##_b, gen_helper_##NAME##_h, \
1347	gen_helper_##NAME##_w, gen_helper_##NAME##_d, \
1348	}; \
1349	if (!opivv_check(s, a)) { \
1350	return false; \
1351	} \
1352	return do_opivv_gvec(s, a, tcg_gen_gvec_##SUF, fns[s->sew]); \
1353	}
1354
1355	GEN_OPIVV_GVEC_TRANS(vadd_vv, add)
1356	GEN_OPIVV_GVEC_TRANS(vsub_vv, sub)
1357
1358	typedef void gen_helper_opivx(TCGv_ptr, TCGv_ptr, TCGv, TCGv_ptr,
1359	TCGv_env, TCGv_i32);
1360
1361	static bool opivx_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, uint32_t vm,
1362	gen_helper_opivx fn, DisasContext s)
1363	{
1364	TCGv_ptr dest, src2, mask;
1365	TCGv src1;
1366	TCGv_i32 desc;
1367	uint32_t data = 0;
1368
1369	dest = tcg_temp_new_ptr();
1370	mask = tcg_temp_new_ptr();
1371	src2 = tcg_temp_new_ptr();
1372	src1 = get_gpr(s, rs1, EXT_SIGN);
1373
1374	data = FIELD_DP32(data, VDATA, VM, vm);
1375	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1376	data = FIELD_DP32(data, VDATA, VTA, s->vta);
1377	data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);
1378	data = FIELD_DP32(data, VDATA, VMA, s->vma);
1379	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
1380	s->cfg_ptr->vlenb, data));
1381
1382	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
1383	tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, vs2));
1384	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
1385
1386	fn(dest, mask, src1, src2, tcg_env, desc);
1387
1388	finalize_rvv_inst(s);
1389	return true;
1390	}
1391
1392	static bool opivx_check(DisasContext s, arg_rmrr a)
1393	{
1394	return require_rvv(s) &&
1395	vext_check_isa_ill(s) &&
1396	vext_check_ss(s, a->rd, a->rs2, a->vm);
1397	}
1398
1399	typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t, TCGv_i64,
1400	uint32_t, uint32_t);
1401
1402	static inline bool
1403	do_opivx_gvec(DisasContext s, arg_rmrr a, GVecGen2sFn *gvec_fn,
1404	gen_helper_opivx *fn)
1405	{
1406	if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
1407	TCGv_i64 src1 = tcg_temp_new_i64();
1408
1409	tcg_gen_ext_tl_i64(src1, get_gpr(s, a->rs1, EXT_SIGN));
1410	gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2),
1411	src1, MAXSZ(s), MAXSZ(s));
1412
1413	finalize_rvv_inst(s);
1414	return true;
1415	}
1416	return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s);
1417	}
1418
1419	/* OPIVX with GVEC IR */
1420	#define GEN_OPIVX_GVEC_TRANS(NAME, SUF) \
1421	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1422	{ \
1423	static gen_helper_opivx * const fns[4] = { \
1424	gen_helper_##NAME##_b, gen_helper_##NAME##_h, \
1425	gen_helper_##NAME##_w, gen_helper_##NAME##_d, \
1426	}; \
1427	if (!opivx_check(s, a)) { \
1428	return false; \
1429	} \
1430	return do_opivx_gvec(s, a, tcg_gen_gvec_##SUF, fns[s->sew]); \
1431	}
1432
1433	GEN_OPIVX_GVEC_TRANS(vadd_vx, adds)
1434	GEN_OPIVX_GVEC_TRANS(vsub_vx, subs)
1435
1436	static void gen_vec_rsub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1437	{
1438	tcg_gen_vec_sub8_i64(d, b, a);
1439	}
1440
1441	static void gen_vec_rsub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1442	{
1443	tcg_gen_vec_sub16_i64(d, b, a);
1444	}
1445
1446	static void gen_rsub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
1447	{
1448	tcg_gen_sub_i32(ret, arg2, arg1);
1449	}
1450
1451	static void gen_rsub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1452	{
1453	tcg_gen_sub_i64(ret, arg2, arg1);
1454	}
1455
1456	static void gen_rsub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
1457	{
1458	tcg_gen_sub_vec(vece, r, b, a);
1459	}
1460
1461	static void tcg_gen_gvec_rsubs(unsigned vece, uint32_t dofs, uint32_t aofs,
1462	TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1463	{
1464	static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 };
1465	static const GVecGen2s rsub_op[4] = {
1466	{ .fni8 = gen_vec_rsub8_i64,
1467	.fniv = gen_rsub_vec,
1468	.fno = gen_helper_vec_rsubs8,
1469	.opt_opc = vecop_list,
1470	.vece = MO_8 },
1471	{ .fni8 = gen_vec_rsub16_i64,
1472	.fniv = gen_rsub_vec,
1473	.fno = gen_helper_vec_rsubs16,
1474	.opt_opc = vecop_list,
1475	.vece = MO_16 },
1476	{ .fni4 = gen_rsub_i32,
1477	.fniv = gen_rsub_vec,
1478	.fno = gen_helper_vec_rsubs32,
1479	.opt_opc = vecop_list,
1480	.vece = MO_32 },
1481	{ .fni8 = gen_rsub_i64,
1482	.fniv = gen_rsub_vec,
1483	.fno = gen_helper_vec_rsubs64,
1484	.opt_opc = vecop_list,
1485	.prefer_i64 = true,
1486	.vece = MO_64 },
1487	};
1488
1489	tcg_debug_assert(vece <= MO_64);
1490	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &rsub_op[vece]);
1491	}
1492
1493	GEN_OPIVX_GVEC_TRANS(vrsub_vx, rsubs)
1494
1495	typedef enum {
1496	IMM_ZX, /* Zero-extended */
1497	IMM_SX, /* Sign-extended */
1498	IMM_TRUNC_SEW, /* Truncate to log(SEW) bits */
1499	IMM_TRUNC_2SEW, /* Truncate to log(2SEW) bits /
1500	} imm_mode_t;
1501
1502	static int64_t extract_imm(DisasContext *s, uint32_t imm, imm_mode_t imm_mode)
1503	{
1504	switch (imm_mode) {
1505	case IMM_ZX:
1506	return extract64(imm, 0, 5);
1507	case IMM_SX:
1508	return sextract64(imm, 0, 5);
1509	case IMM_TRUNC_SEW:
1510	return extract64(imm, 0, s->sew + 3);
1511	case IMM_TRUNC_2SEW:
1512	return extract64(imm, 0, s->sew + 4);
1513	default:
1514	g_assert_not_reached();
1515	}
1516	}
1517
1518	static bool opivi_trans(uint32_t vd, uint32_t imm, uint32_t vs2, uint32_t vm,
1519	gen_helper_opivx fn, DisasContext s,
1520	imm_mode_t imm_mode)
1521	{
1522	TCGv_ptr dest, src2, mask;
1523	TCGv src1;
1524	TCGv_i32 desc;
1525	uint32_t data = 0;
1526
1527	dest = tcg_temp_new_ptr();
1528	mask = tcg_temp_new_ptr();
1529	src2 = tcg_temp_new_ptr();
1530	src1 = tcg_constant_tl(extract_imm(s, imm, imm_mode));
1531
1532	data = FIELD_DP32(data, VDATA, VM, vm);
1533	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1534	data = FIELD_DP32(data, VDATA, VTA, s->vta);
1535	data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);
1536	data = FIELD_DP32(data, VDATA, VMA, s->vma);
1537	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
1538	s->cfg_ptr->vlenb, data));
1539
1540	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
1541	tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, vs2));
1542	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
1543
1544	fn(dest, mask, src1, src2, tcg_env, desc);
1545
1546	finalize_rvv_inst(s);
1547	return true;
1548	}
1549
1550	typedef void GVecGen2iFn(unsigned, uint32_t, uint32_t, int64_t,
1551	uint32_t, uint32_t);
1552
1553	static inline bool
1554	do_opivi_gvec(DisasContext s, arg_rmrr a, GVecGen2iFn *gvec_fn,
1555	gen_helper_opivx *fn, imm_mode_t imm_mode)
1556	{
1557	if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
1558	gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2),
1559	extract_imm(s, a->rs1, imm_mode), MAXSZ(s), MAXSZ(s));
1560	finalize_rvv_inst(s);
1561	return true;
1562	}
1563	return opivi_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s, imm_mode);
1564	}
1565
1566	/* OPIVI with GVEC IR */
1567	#define GEN_OPIVI_GVEC_TRANS(NAME, IMM_MODE, OPIVX, SUF) \
1568	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1569	{ \
1570	static gen_helper_opivx * const fns[4] = { \
1571	gen_helper_##OPIVX##_b, gen_helper_##OPIVX##_h, \
1572	gen_helper_##OPIVX##_w, gen_helper_##OPIVX##_d, \
1573	}; \
1574	if (!opivx_check(s, a)) { \
1575	return false; \
1576	} \
1577	return do_opivi_gvec(s, a, tcg_gen_gvec_##SUF, \
1578	fns[s->sew], IMM_MODE); \
1579	}
1580
1581	GEN_OPIVI_GVEC_TRANS(vadd_vi, IMM_SX, vadd_vx, addi)
1582
1583	static void tcg_gen_gvec_rsubi(unsigned vece, uint32_t dofs, uint32_t aofs,
1584	int64_t c, uint32_t oprsz, uint32_t maxsz)
1585	{
1586	TCGv_i64 tmp = tcg_constant_i64(c);
1587	tcg_gen_gvec_rsubs(vece, dofs, aofs, tmp, oprsz, maxsz);
1588	}
1589
1590	GEN_OPIVI_GVEC_TRANS(vrsub_vi, IMM_SX, vrsub_vx, rsubi)
1591
1592	/* Vector Widening Integer Add/Subtract */
1593
1594	/* OPIVV with WIDEN */
1595	static bool opivv_widen_check(DisasContext s, arg_rmrr a)
1596	{
1597	return require_rvv(s) &&
1598	vext_check_isa_ill(s) &&
1599	vext_check_dss(s, a->rd, a->rs1, a->rs2, a->vm);
1600	}
1601
1602	/* OPIVV with overwrite and WIDEN */
1603	static bool opivv_overwrite_widen_check(DisasContext s, arg_rmrr a)
1604	{
1605	return require_rvv(s) &&
1606	vext_check_isa_ill(s) &&
1607	vext_check_dss(s, a->rd, a->rs1, a->rs2, a->vm) &&
1608	vext_check_input_eew(s, a->rd, s->sew + 1, a->rs1, s->sew, a->vm) &&
1609	vext_check_input_eew(s, a->rd, s->sew + 1, a->rs2, s->sew, a->vm);
1610	}
1611
1612	static bool do_opivv_widen(DisasContext s, arg_rmrr a,
1613	gen_helper_gvec_4_ptr *fn,
1614	bool (checkfn)(DisasContext , arg_rmrr *))
1615	{
1616	if (checkfn(s, a)) {
1617	uint32_t data = 0;
1618
1619	data = FIELD_DP32(data, VDATA, VM, a->vm);
1620	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1621	data = FIELD_DP32(data, VDATA, VTA, s->vta);
1622	data = FIELD_DP32(data, VDATA, VMA, s->vma);
1623	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
1624	vreg_ofs(s, a->rs1),
1625	vreg_ofs(s, a->rs2),
1626	tcg_env, s->cfg_ptr->vlenb,
1627	s->cfg_ptr->vlenb,
1628	data, fn);
1629	finalize_rvv_inst(s);
1630	return true;
1631	}
1632	return false;
1633	}
1634
1635	#define GEN_OPIVV_WIDEN_TRANS(NAME, CHECK) \
1636	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1637	{ \
1638	static gen_helper_gvec_4_ptr * const fns[3] = { \
1639	gen_helper_##NAME##_b, \
1640	gen_helper_##NAME##_h, \
1641	gen_helper_##NAME##_w \
1642	}; \
1643	return do_opivv_widen(s, a, fns[s->sew], CHECK); \
1644	}
1645
1646	GEN_OPIVV_WIDEN_TRANS(vwaddu_vv, opivv_widen_check)
1647	GEN_OPIVV_WIDEN_TRANS(vwadd_vv, opivv_widen_check)
1648	GEN_OPIVV_WIDEN_TRANS(vwsubu_vv, opivv_widen_check)
1649	GEN_OPIVV_WIDEN_TRANS(vwsub_vv, opivv_widen_check)
1650
1651	/* OPIVX with WIDEN */
1652	static bool opivx_widen_check(DisasContext s, arg_rmrr a)
1653	{
1654	return require_rvv(s) &&
1655	vext_check_isa_ill(s) &&
1656	vext_check_ds(s, a->rd, a->rs2, a->vm);
1657	}
1658
1659	static bool opivx_overwrite_widen_check(DisasContext s, arg_rmrr a)
1660	{
1661	return require_rvv(s) &&
1662	vext_check_isa_ill(s) &&
1663	vext_check_ds(s, a->rd, a->rs2, a->vm) &&
1664	vext_check_input_eew(s, a->rd, s->sew + 1, a->rs2, s->sew, a->vm);
1665	}
1666
1667	#define GEN_OPIVX_WIDEN_TRANS(NAME, CHECK) \
1668	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1669	{ \
1670	if (CHECK(s, a)) { \
1671	static gen_helper_opivx * const fns[3] = { \
1672	gen_helper_##NAME##_b, \
1673	gen_helper_##NAME##_h, \
1674	gen_helper_##NAME##_w \
1675	}; \
1676	return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew], s); \
1677	} \
1678	return false; \
1679	}
1680
1681	GEN_OPIVX_WIDEN_TRANS(vwaddu_vx, opivx_widen_check)
1682	GEN_OPIVX_WIDEN_TRANS(vwadd_vx, opivx_widen_check)
1683	GEN_OPIVX_WIDEN_TRANS(vwsubu_vx, opivx_widen_check)
1684	GEN_OPIVX_WIDEN_TRANS(vwsub_vx, opivx_widen_check)
1685
1686	/* WIDEN OPIVV with WIDEN */
1687	static bool opiwv_widen_check(DisasContext s, arg_rmrr a)
1688	{
1689	return require_rvv(s) &&
1690	vext_check_isa_ill(s) &&
1691	vext_check_dds(s, a->rd, a->rs1, a->rs2, a->vm);
1692	}
1693
1694	static bool do_opiwv_widen(DisasContext s, arg_rmrr a,
1695	gen_helper_gvec_4_ptr *fn)
1696	{
1697	if (opiwv_widen_check(s, a)) {
1698	uint32_t data = 0;
1699
1700	data = FIELD_DP32(data, VDATA, VM, a->vm);
1701	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1702	data = FIELD_DP32(data, VDATA, VTA, s->vta);
1703	data = FIELD_DP32(data, VDATA, VMA, s->vma);
1704	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
1705	vreg_ofs(s, a->rs1),
1706	vreg_ofs(s, a->rs2),
1707	tcg_env, s->cfg_ptr->vlenb,
1708	s->cfg_ptr->vlenb, data, fn);
1709	finalize_rvv_inst(s);
1710	return true;
1711	}
1712	return false;
1713	}
1714
1715	#define GEN_OPIWV_WIDEN_TRANS(NAME) \
1716	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1717	{ \
1718	static gen_helper_gvec_4_ptr * const fns[3] = { \
1719	gen_helper_##NAME##_b, \
1720	gen_helper_##NAME##_h, \
1721	gen_helper_##NAME##_w \
1722	}; \
1723	return do_opiwv_widen(s, a, fns[s->sew]); \
1724	}
1725
1726	GEN_OPIWV_WIDEN_TRANS(vwaddu_wv)
1727	GEN_OPIWV_WIDEN_TRANS(vwadd_wv)
1728	GEN_OPIWV_WIDEN_TRANS(vwsubu_wv)
1729	GEN_OPIWV_WIDEN_TRANS(vwsub_wv)
1730
1731	/* WIDEN OPIVX with WIDEN */
1732	static bool opiwx_widen_check(DisasContext s, arg_rmrr a)
1733	{
1734	return require_rvv(s) &&
1735	vext_check_isa_ill(s) &&
1736	vext_check_dd(s, a->rd, a->rs2, a->vm);
1737	}
1738
1739	static bool do_opiwx_widen(DisasContext s, arg_rmrr a,
1740	gen_helper_opivx *fn)
1741	{
1742	if (opiwx_widen_check(s, a)) {
1743	return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s);
1744	}
1745	return false;
1746	}
1747
1748	#define GEN_OPIWX_WIDEN_TRANS(NAME) \
1749	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1750	{ \
1751	static gen_helper_opivx * const fns[3] = { \
1752	gen_helper_##NAME##_b, \
1753	gen_helper_##NAME##_h, \
1754	gen_helper_##NAME##_w \
1755	}; \
1756	return do_opiwx_widen(s, a, fns[s->sew]); \
1757	}
1758
1759	GEN_OPIWX_WIDEN_TRANS(vwaddu_wx)
1760	GEN_OPIWX_WIDEN_TRANS(vwadd_wx)
1761	GEN_OPIWX_WIDEN_TRANS(vwsubu_wx)
1762	GEN_OPIWX_WIDEN_TRANS(vwsub_wx)
1763
1764	static bool opivv_trans(uint32_t vd, uint32_t vs1, uint32_t vs2, uint32_t vm,
1765	gen_helper_gvec_4_ptr fn, DisasContext s)
1766	{
1767	uint32_t data = 0;
1768
1769	data = FIELD_DP32(data, VDATA, VM, vm);
1770	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1771	data = FIELD_DP32(data, VDATA, VTA, s->vta);
1772	data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);
1773	data = FIELD_DP32(data, VDATA, VMA, s->vma);
1774	tcg_gen_gvec_4_ptr(vreg_ofs(s, vd), vreg_ofs(s, 0), vreg_ofs(s, vs1),
1775	vreg_ofs(s, vs2), tcg_env, s->cfg_ptr->vlenb,
1776	s->cfg_ptr->vlenb, data, fn);
1777	finalize_rvv_inst(s);
1778	return true;
1779	}
1780
1781	/* Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions */
1782	/* OPIVV without GVEC IR */
1783	#define GEN_OPIVV_TRANS(NAME, CHECK) \
1784	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1785	{ \
1786	if (CHECK(s, a)) { \
1787	static gen_helper_gvec_4_ptr * const fns[4] = { \
1788	gen_helper_##NAME##_b, gen_helper_##NAME##_h, \
1789	gen_helper_##NAME##_w, gen_helper_##NAME##_d, \
1790	}; \
1791	return opivv_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew], s);\
1792	} \
1793	return false; \
1794	}
1795
1796	/*
1797	* For vadc and vsbc, an illegal instruction exception is raised if the
1798	* destination vector register is v0 and LMUL > 1. (Section 11.4)
1799	*/
1800	static bool opivv_vadc_check(DisasContext s, arg_rmrr a)
1801	{
1802	return require_rvv(s) &&
1803	vext_check_isa_ill(s) &&
1804	(a->rd != 0) &&
1805	vext_check_sss(s, a->rd, a->rs1, a->rs2, a->vm);
1806	}
1807
1808	GEN_OPIVV_TRANS(vadc_vvm, opivv_vadc_check)
1809	GEN_OPIVV_TRANS(vsbc_vvm, opivv_vadc_check)
1810
1811	/*
1812	* For vmadc and vmsbc, an illegal instruction exception is raised if the
1813	* destination vector register overlaps a source vector register group.
1814	*/
1815	static bool opivv_vmadc_check(DisasContext s, arg_rmrr a)
1816	{
1817	return require_rvv(s) &&
1818	vext_check_isa_ill(s) &&
1819	vext_check_mss(s, a->rd, a->rs1, a->rs2);
1820	}
1821
1822	GEN_OPIVV_TRANS(vmadc_vvm, opivv_vmadc_check)
1823	GEN_OPIVV_TRANS(vmsbc_vvm, opivv_vmadc_check)
1824
1825	static bool opivx_vadc_check(DisasContext s, arg_rmrr a)
1826	{
1827	return require_rvv(s) &&
1828	vext_check_isa_ill(s) &&
1829	(a->rd != 0) &&
1830	vext_check_ss(s, a->rd, a->rs2, a->vm);
1831	}
1832
1833	/* OPIVX without GVEC IR */
1834	#define GEN_OPIVX_TRANS(NAME, CHECK) \
1835	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1836	{ \
1837	if (CHECK(s, a)) { \
1838	static gen_helper_opivx * const fns[4] = { \
1839	gen_helper_##NAME##_b, gen_helper_##NAME##_h, \
1840	gen_helper_##NAME##_w, gen_helper_##NAME##_d, \
1841	}; \
1842	\
1843	return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew], s);\
1844	} \
1845	return false; \
1846	}
1847
1848	GEN_OPIVX_TRANS(vadc_vxm, opivx_vadc_check)
1849	GEN_OPIVX_TRANS(vsbc_vxm, opivx_vadc_check)
1850
1851	static bool opivx_vmadc_check(DisasContext s, arg_rmrr a)
1852	{
1853	return require_rvv(s) &&
1854	vext_check_isa_ill(s) &&
1855	vext_check_ms(s, a->rd, a->rs2);
1856	}
1857
1858	GEN_OPIVX_TRANS(vmadc_vxm, opivx_vmadc_check)
1859	GEN_OPIVX_TRANS(vmsbc_vxm, opivx_vmadc_check)
1860
1861	/* OPIVI without GVEC IR */
1862	#define GEN_OPIVI_TRANS(NAME, IMM_MODE, OPIVX, CHECK) \
1863	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1864	{ \
1865	if (CHECK(s, a)) { \
1866	static gen_helper_opivx * const fns[4] = { \
1867	gen_helper_##OPIVX##_b, gen_helper_##OPIVX##_h, \
1868	gen_helper_##OPIVX##_w, gen_helper_##OPIVX##_d, \
1869	}; \
1870	return opivi_trans(a->rd, a->rs1, a->rs2, a->vm, \
1871	fns[s->sew], s, IMM_MODE); \
1872	} \
1873	return false; \
1874	}
1875
1876	GEN_OPIVI_TRANS(vadc_vim, IMM_SX, vadc_vxm, opivx_vadc_check)
1877	GEN_OPIVI_TRANS(vmadc_vim, IMM_SX, vmadc_vxm, opivx_vmadc_check)
1878
1879	/* Vector Bitwise Logical Instructions */
1880	GEN_OPIVV_GVEC_TRANS(vand_vv, and)
1881	GEN_OPIVV_GVEC_TRANS(vor_vv, or)
1882	GEN_OPIVV_GVEC_TRANS(vxor_vv, xor)
1883	GEN_OPIVX_GVEC_TRANS(vand_vx, ands)
1884	GEN_OPIVX_GVEC_TRANS(vor_vx, ors)
1885	GEN_OPIVX_GVEC_TRANS(vxor_vx, xors)
1886	GEN_OPIVI_GVEC_TRANS(vand_vi, IMM_SX, vand_vx, andi)
1887	GEN_OPIVI_GVEC_TRANS(vor_vi, IMM_SX, vor_vx, ori)
1888	GEN_OPIVI_GVEC_TRANS(vxor_vi, IMM_SX, vxor_vx, xori)
1889
1890	/* Vector Single-Width Bit Shift Instructions */
1891	GEN_OPIVV_GVEC_TRANS(vsll_vv, shlv)
1892	GEN_OPIVV_GVEC_TRANS(vsrl_vv, shrv)
1893	GEN_OPIVV_GVEC_TRANS(vsra_vv, sarv)
1894
1895	typedef void GVecGen2sFn32(unsigned, uint32_t, uint32_t, TCGv_i32,
1896	uint32_t, uint32_t);
1897
1898	static inline bool
1899	do_opivx_gvec_shift(DisasContext s, arg_rmrr a, GVecGen2sFn32 *gvec_fn,
1900	gen_helper_opivx *fn)
1901	{
1902	if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
1903	TCGv_i32 src1 = tcg_temp_new_i32();
1904
1905	tcg_gen_trunc_tl_i32(src1, get_gpr(s, a->rs1, EXT_NONE));
1906	tcg_gen_extract_i32(src1, src1, 0, s->sew + 3);
1907	gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2),
1908	src1, MAXSZ(s), MAXSZ(s));
1909
1910	finalize_rvv_inst(s);
1911	return true;
1912	}
1913	return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s);
1914	}
1915
1916	#define GEN_OPIVX_GVEC_SHIFT_TRANS(NAME, SUF) \
1917	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1918	{ \
1919	static gen_helper_opivx * const fns[4] = { \
1920	gen_helper_##NAME##_b, gen_helper_##NAME##_h, \
1921	gen_helper_##NAME##_w, gen_helper_##NAME##_d, \
1922	}; \
1923	if (!opivx_check(s, a)) { \
1924	return false; \
1925	} \
1926	return do_opivx_gvec_shift(s, a, tcg_gen_gvec_##SUF, fns[s->sew]); \
1927	}
1928
1929	GEN_OPIVX_GVEC_SHIFT_TRANS(vsll_vx, shls)
1930	GEN_OPIVX_GVEC_SHIFT_TRANS(vsrl_vx, shrs)
1931	GEN_OPIVX_GVEC_SHIFT_TRANS(vsra_vx, sars)
1932
1933	GEN_OPIVI_GVEC_TRANS(vsll_vi, IMM_TRUNC_SEW, vsll_vx, shli)
1934	GEN_OPIVI_GVEC_TRANS(vsrl_vi, IMM_TRUNC_SEW, vsrl_vx, shri)
1935	GEN_OPIVI_GVEC_TRANS(vsra_vi, IMM_TRUNC_SEW, vsra_vx, sari)
1936
1937	/* Vector Narrowing Integer Right Shift Instructions */
1938	static bool opiwv_narrow_check(DisasContext s, arg_rmrr a)
1939	{
1940	return require_rvv(s) &&
1941	vext_check_isa_ill(s) &&
1942	vext_check_sds(s, a->rd, a->rs1, a->rs2, a->vm);
1943	}
1944
1945	/* OPIVV with NARROW */
1946	#define GEN_OPIWV_NARROW_TRANS(NAME) \
1947	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1948	{ \
1949	if (opiwv_narrow_check(s, a)) { \
1950	uint32_t data = 0; \
1951	static gen_helper_gvec_4_ptr * const fns[3] = { \
1952	gen_helper_##NAME##_b, \
1953	gen_helper_##NAME##_h, \
1954	gen_helper_##NAME##_w, \
1955	}; \
1956	\
1957	data = FIELD_DP32(data, VDATA, VM, a->vm); \
1958	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
1959	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
1960	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
1961	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
1962	vreg_ofs(s, a->rs1), \
1963	vreg_ofs(s, a->rs2), tcg_env, \
1964	s->cfg_ptr->vlenb, \
1965	s->cfg_ptr->vlenb, data, \
1966	fns[s->sew]); \
1967	finalize_rvv_inst(s); \
1968	return true; \
1969	} \
1970	return false; \
1971	}
1972	GEN_OPIWV_NARROW_TRANS(vnsra_wv)
1973	GEN_OPIWV_NARROW_TRANS(vnsrl_wv)
1974
1975	static bool opiwx_narrow_check(DisasContext s, arg_rmrr a)
1976	{
1977	return require_rvv(s) &&
1978	vext_check_isa_ill(s) &&
1979	vext_check_sd(s, a->rd, a->rs2, a->vm);
1980	}
1981
1982	/* OPIVX with NARROW */
1983	#define GEN_OPIWX_NARROW_TRANS(NAME) \
1984	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
1985	{ \
1986	if (opiwx_narrow_check(s, a)) { \
1987	static gen_helper_opivx * const fns[3] = { \
1988	gen_helper_##NAME##_b, \
1989	gen_helper_##NAME##_h, \
1990	gen_helper_##NAME##_w, \
1991	}; \
1992	return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew], s);\
1993	} \
1994	return false; \
1995	}
1996
1997	GEN_OPIWX_NARROW_TRANS(vnsra_wx)
1998	GEN_OPIWX_NARROW_TRANS(vnsrl_wx)
1999
2000	/* OPIWI with NARROW */
2001	#define GEN_OPIWI_NARROW_TRANS(NAME, IMM_MODE, OPIVX) \
2002	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2003	{ \
2004	if (opiwx_narrow_check(s, a)) { \
2005	static gen_helper_opivx * const fns[3] = { \
2006	gen_helper_##OPIVX##_b, \
2007	gen_helper_##OPIVX##_h, \
2008	gen_helper_##OPIVX##_w, \
2009	}; \
2010	return opivi_trans(a->rd, a->rs1, a->rs2, a->vm, \
2011	fns[s->sew], s, IMM_MODE); \
2012	} \
2013	return false; \
2014	}
2015
2016	GEN_OPIWI_NARROW_TRANS(vnsra_wi, IMM_ZX, vnsra_wx)
2017	GEN_OPIWI_NARROW_TRANS(vnsrl_wi, IMM_ZX, vnsrl_wx)
2018
2019	/* Vector Integer Comparison Instructions */
2020	/*
2021	* For all comparison instructions, an illegal instruction exception is raised
2022	* if the destination vector register overlaps a source vector register group
2023	* and LMUL > 1.
2024	*/
2025	static bool opivv_cmp_check(DisasContext s, arg_rmrr a)
2026	{
2027	return require_rvv(s) &&
2028	vext_check_isa_ill(s) &&
2029	vext_check_mss(s, a->rd, a->rs1, a->rs2);
2030	}
2031
2032	GEN_OPIVV_TRANS(vmseq_vv, opivv_cmp_check)
2033	GEN_OPIVV_TRANS(vmsne_vv, opivv_cmp_check)
2034	GEN_OPIVV_TRANS(vmsltu_vv, opivv_cmp_check)
2035	GEN_OPIVV_TRANS(vmslt_vv, opivv_cmp_check)
2036	GEN_OPIVV_TRANS(vmsleu_vv, opivv_cmp_check)
2037	GEN_OPIVV_TRANS(vmsle_vv, opivv_cmp_check)
2038
2039	static bool opivx_cmp_check(DisasContext s, arg_rmrr a)
2040	{
2041	return require_rvv(s) &&
2042	vext_check_isa_ill(s) &&
2043	vext_check_ms(s, a->rd, a->rs2);
2044	}
2045
2046	GEN_OPIVX_TRANS(vmseq_vx, opivx_cmp_check)
2047	GEN_OPIVX_TRANS(vmsne_vx, opivx_cmp_check)
2048	GEN_OPIVX_TRANS(vmsltu_vx, opivx_cmp_check)
2049	GEN_OPIVX_TRANS(vmslt_vx, opivx_cmp_check)
2050	GEN_OPIVX_TRANS(vmsleu_vx, opivx_cmp_check)
2051	GEN_OPIVX_TRANS(vmsle_vx, opivx_cmp_check)
2052	GEN_OPIVX_TRANS(vmsgtu_vx, opivx_cmp_check)
2053	GEN_OPIVX_TRANS(vmsgt_vx, opivx_cmp_check)
2054
2055	GEN_OPIVI_TRANS(vmseq_vi, IMM_SX, vmseq_vx, opivx_cmp_check)
2056	GEN_OPIVI_TRANS(vmsne_vi, IMM_SX, vmsne_vx, opivx_cmp_check)
2057	GEN_OPIVI_TRANS(vmsleu_vi, IMM_SX, vmsleu_vx, opivx_cmp_check)
2058	GEN_OPIVI_TRANS(vmsle_vi, IMM_SX, vmsle_vx, opivx_cmp_check)
2059	GEN_OPIVI_TRANS(vmsgtu_vi, IMM_SX, vmsgtu_vx, opivx_cmp_check)
2060	GEN_OPIVI_TRANS(vmsgt_vi, IMM_SX, vmsgt_vx, opivx_cmp_check)
2061
2062	/* Vector Integer Min/Max Instructions */
2063	GEN_OPIVV_GVEC_TRANS(vminu_vv, umin)
2064	GEN_OPIVV_GVEC_TRANS(vmin_vv, smin)
2065	GEN_OPIVV_GVEC_TRANS(vmaxu_vv, umax)
2066	GEN_OPIVV_GVEC_TRANS(vmax_vv, smax)
2067	GEN_OPIVX_TRANS(vminu_vx, opivx_check)
2068	GEN_OPIVX_TRANS(vmin_vx, opivx_check)
2069	GEN_OPIVX_TRANS(vmaxu_vx, opivx_check)
2070	GEN_OPIVX_TRANS(vmax_vx, opivx_check)
2071
2072	/* Vector Single-Width Integer Multiply Instructions */
2073
2074	static bool vmulh_vv_check(DisasContext s, arg_rmrr a)
2075	{
2076	/*
2077	* All Zve* extensions support all vector integer instructions,
2078	* except that the vmulh integer multiply variants
2079	* that return the high word of the product
2080	* (vmulh.vv, vmulh.vx, vmulhu.vv, vmulhu.vx, vmulhsu.vv, vmulhsu.vx)
2081	* are not included for EEW=64 in Zve64*. (Section 18.2)
2082	*/
2083	return opivv_check(s, a) &&
2084	(!has_ext(s, RVV) ? s->sew != MO_64 : true);
2085	}
2086
2087	static bool vmulh_vx_check(DisasContext s, arg_rmrr a)
2088	{
2089	/*
2090	* All Zve* extensions support all vector integer instructions,
2091	* except that the vmulh integer multiply variants
2092	* that return the high word of the product
2093	* (vmulh.vv, vmulh.vx, vmulhu.vv, vmulhu.vx, vmulhsu.vv, vmulhsu.vx)
2094	* are not included for EEW=64 in Zve64*. (Section 18.2)
2095	*/
2096	return opivx_check(s, a) &&
2097	(!has_ext(s, RVV) ? s->sew != MO_64 : true);
2098	}
2099
2100	GEN_OPIVV_GVEC_TRANS(vmul_vv, mul)
2101	GEN_OPIVV_TRANS(vmulh_vv, vmulh_vv_check)
2102	GEN_OPIVV_TRANS(vmulhu_vv, vmulh_vv_check)
2103	GEN_OPIVV_TRANS(vmulhsu_vv, vmulh_vv_check)
2104	GEN_OPIVX_GVEC_TRANS(vmul_vx, muls)
2105	GEN_OPIVX_TRANS(vmulh_vx, vmulh_vx_check)
2106	GEN_OPIVX_TRANS(vmulhu_vx, vmulh_vx_check)
2107	GEN_OPIVX_TRANS(vmulhsu_vx, vmulh_vx_check)
2108
2109	/* Vector Integer Divide Instructions */
2110	GEN_OPIVV_TRANS(vdivu_vv, opivv_check)
2111	GEN_OPIVV_TRANS(vdiv_vv, opivv_check)
2112	GEN_OPIVV_TRANS(vremu_vv, opivv_check)
2113	GEN_OPIVV_TRANS(vrem_vv, opivv_check)
2114	GEN_OPIVX_TRANS(vdivu_vx, opivx_check)
2115	GEN_OPIVX_TRANS(vdiv_vx, opivx_check)
2116	GEN_OPIVX_TRANS(vremu_vx, opivx_check)
2117	GEN_OPIVX_TRANS(vrem_vx, opivx_check)
2118
2119	/* Vector Widening Integer Multiply Instructions */
2120	GEN_OPIVV_WIDEN_TRANS(vwmul_vv, opivv_widen_check)
2121	GEN_OPIVV_WIDEN_TRANS(vwmulu_vv, opivv_widen_check)
2122	GEN_OPIVV_WIDEN_TRANS(vwmulsu_vv, opivv_widen_check)
2123	GEN_OPIVX_WIDEN_TRANS(vwmul_vx, opivx_widen_check)
2124	GEN_OPIVX_WIDEN_TRANS(vwmulu_vx, opivx_widen_check)
2125	GEN_OPIVX_WIDEN_TRANS(vwmulsu_vx, opivx_widen_check)
2126
2127	/* Vector Single-Width Integer Multiply-Add Instructions */
2128	GEN_OPIVV_TRANS(vmacc_vv, opivv_check)
2129	GEN_OPIVV_TRANS(vnmsac_vv, opivv_check)
2130	GEN_OPIVV_TRANS(vmadd_vv, opivv_check)
2131	GEN_OPIVV_TRANS(vnmsub_vv, opivv_check)
2132	GEN_OPIVX_TRANS(vmacc_vx, opivx_check)
2133	GEN_OPIVX_TRANS(vnmsac_vx, opivx_check)
2134	GEN_OPIVX_TRANS(vmadd_vx, opivx_check)
2135	GEN_OPIVX_TRANS(vnmsub_vx, opivx_check)
2136
2137	/* Vector Widening Integer Multiply-Add Instructions */
2138	GEN_OPIVV_WIDEN_TRANS(vwmaccu_vv, opivv_overwrite_widen_check)
2139	GEN_OPIVV_WIDEN_TRANS(vwmacc_vv, opivv_overwrite_widen_check)
2140	GEN_OPIVV_WIDEN_TRANS(vwmaccsu_vv, opivv_overwrite_widen_check)
2141	GEN_OPIVX_WIDEN_TRANS(vwmaccu_vx, opivx_overwrite_widen_check)
2142	GEN_OPIVX_WIDEN_TRANS(vwmacc_vx, opivx_overwrite_widen_check)
2143	GEN_OPIVX_WIDEN_TRANS(vwmaccsu_vx, opivx_overwrite_widen_check)
2144	GEN_OPIVX_WIDEN_TRANS(vwmaccus_vx, opivx_overwrite_widen_check)
2145
2146	/* Vector Integer Merge and Move Instructions */
2147	static bool trans_vmv_v_v(DisasContext s, arg_vmv_v_v a)
2148	{
2149	if (require_rvv(s) &&
2150	vext_check_isa_ill(s) &&
2151	/* vmv.v.v has rs2 = 0 and vm = 1 */
2152	vext_check_sss(s, a->rd, a->rs1, 0, 1)) {
2153	if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
2154	tcg_gen_gvec_mov(s->sew, vreg_ofs(s, a->rd),
2155	vreg_ofs(s, a->rs1),
2156	MAXSZ(s), MAXSZ(s));
2157	} else {
2158	uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul);
2159	data = FIELD_DP32(data, VDATA, VTA, s->vta);
2160	static gen_helper_gvec_2_ptr * const fns[4] = {
2161	gen_helper_vmv_v_v_b, gen_helper_vmv_v_v_h,
2162	gen_helper_vmv_v_v_w, gen_helper_vmv_v_v_d,
2163	};
2164
2165	tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1),
2166	tcg_env, s->cfg_ptr->vlenb,
2167	s->cfg_ptr->vlenb, data,
2168	fns[s->sew]);
2169	}
2170	finalize_rvv_inst(s);
2171	return true;
2172	}
2173	return false;
2174	}
2175
2176	typedef void gen_helper_vmv_vx(TCGv_ptr, TCGv_i64, TCGv_env, TCGv_i32);
2177	static bool trans_vmv_v_x(DisasContext s, arg_vmv_v_x a)
2178	{
2179	if (require_rvv(s) &&
2180	vext_check_isa_ill(s) &&
2181	/* vmv.v.x has rs2 = 0 and vm = 1 */
2182	vext_check_ss(s, a->rd, 0, 1)) {
2183	TCGv s1;
2184
2185	s1 = get_gpr(s, a->rs1, EXT_SIGN);
2186
2187	if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
2188	if (get_xl(s) == MXL_RV32 && s->sew == MO_64) {
2189	TCGv_i64 s1_i64 = tcg_temp_new_i64();
2190	tcg_gen_ext_tl_i64(s1_i64, s1);
2191	tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd),
2192	MAXSZ(s), MAXSZ(s), s1_i64);
2193	} else {
2194	tcg_gen_gvec_dup_tl(s->sew, vreg_ofs(s, a->rd),
2195	MAXSZ(s), MAXSZ(s), s1);
2196	}
2197	} else {
2198	TCGv_i32 desc;
2199	TCGv_i64 s1_i64 = tcg_temp_new_i64();
2200	TCGv_ptr dest = tcg_temp_new_ptr();
2201	uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul);
2202	data = FIELD_DP32(data, VDATA, VTA, s->vta);
2203	static gen_helper_vmv_vx * const fns[4] = {
2204	gen_helper_vmv_v_x_b, gen_helper_vmv_v_x_h,
2205	gen_helper_vmv_v_x_w, gen_helper_vmv_v_x_d,
2206	};
2207
2208	tcg_gen_ext_tl_i64(s1_i64, s1);
2209	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
2210	s->cfg_ptr->vlenb, data));
2211	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd));
2212	fns[s->sew](dest, s1_i64, tcg_env, desc);
2213	}
2214
2215	finalize_rvv_inst(s);
2216	return true;
2217	}
2218	return false;
2219	}
2220
2221	static bool trans_vmv_v_i(DisasContext s, arg_vmv_v_i a)
2222	{
2223	if (require_rvv(s) &&
2224	vext_check_isa_ill(s) &&
2225	/* vmv.v.i has rs2 = 0 and vm = 1 */
2226	vext_check_ss(s, a->rd, 0, 1)) {
2227	int64_t simm = sextract64(a->rs1, 0, 5);
2228	if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
2229	tcg_gen_gvec_dup_imm(s->sew, vreg_ofs(s, a->rd),
2230	MAXSZ(s), MAXSZ(s), simm);
2231	} else {
2232	TCGv_i32 desc;
2233	TCGv_i64 s1;
2234	TCGv_ptr dest;
2235	uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul);
2236	data = FIELD_DP32(data, VDATA, VTA, s->vta);
2237	static gen_helper_vmv_vx * const fns[4] = {
2238	gen_helper_vmv_v_x_b, gen_helper_vmv_v_x_h,
2239	gen_helper_vmv_v_x_w, gen_helper_vmv_v_x_d,
2240	};
2241
2242	s1 = tcg_constant_i64(simm);
2243	dest = tcg_temp_new_ptr();
2244	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
2245	s->cfg_ptr->vlenb, data));
2246	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd));
2247	fns[s->sew](dest, s1, tcg_env, desc);
2248	}
2249	finalize_rvv_inst(s);
2250	return true;
2251	}
2252	return false;
2253	}
2254
2255	GEN_OPIVV_TRANS(vmerge_vvm, opivv_vadc_check)
2256	GEN_OPIVX_TRANS(vmerge_vxm, opivx_vadc_check)
2257	GEN_OPIVI_TRANS(vmerge_vim, IMM_SX, vmerge_vxm, opivx_vadc_check)
2258
2259	/*
2260	*** Vector Fixed-Point Arithmetic Instructions
2261	*/
2262
2263	/* Vector Single-Width Saturating Add and Subtract */
2264	GEN_OPIVV_TRANS(vsaddu_vv, opivv_check)
2265	GEN_OPIVV_TRANS(vsadd_vv, opivv_check)
2266	GEN_OPIVV_TRANS(vssubu_vv, opivv_check)
2267	GEN_OPIVV_TRANS(vssub_vv, opivv_check)
2268	GEN_OPIVX_TRANS(vsaddu_vx, opivx_check)
2269	GEN_OPIVX_TRANS(vsadd_vx, opivx_check)
2270	GEN_OPIVX_TRANS(vssubu_vx, opivx_check)
2271	GEN_OPIVX_TRANS(vssub_vx, opivx_check)
2272	GEN_OPIVI_TRANS(vsaddu_vi, IMM_SX, vsaddu_vx, opivx_check)
2273	GEN_OPIVI_TRANS(vsadd_vi, IMM_SX, vsadd_vx, opivx_check)
2274
2275	/* Vector Single-Width Averaging Add and Subtract */
2276	GEN_OPIVV_TRANS(vaadd_vv, opivv_check)
2277	GEN_OPIVV_TRANS(vaaddu_vv, opivv_check)
2278	GEN_OPIVV_TRANS(vasub_vv, opivv_check)
2279	GEN_OPIVV_TRANS(vasubu_vv, opivv_check)
2280	GEN_OPIVX_TRANS(vaadd_vx, opivx_check)
2281	GEN_OPIVX_TRANS(vaaddu_vx, opivx_check)
2282	GEN_OPIVX_TRANS(vasub_vx, opivx_check)
2283	GEN_OPIVX_TRANS(vasubu_vx, opivx_check)
2284
2285	/* Vector Single-Width Fractional Multiply with Rounding and Saturation */
2286
2287	static bool vsmul_vv_check(DisasContext s, arg_rmrr a)
2288	{
2289	/*
2290	* All Zve* extensions support all vector fixed-point arithmetic
2291	* instructions, except that vsmul.vv and vsmul.vx are not supported
2292	* for EEW=64 in Zve64*. (Section 18.2)
2293	*/
2294	return opivv_check(s, a) &&
2295	(!has_ext(s, RVV) ? s->sew != MO_64 : true);
2296	}
2297
2298	static bool vsmul_vx_check(DisasContext s, arg_rmrr a)
2299	{
2300	/*
2301	* All Zve* extensions support all vector fixed-point arithmetic
2302	* instructions, except that vsmul.vv and vsmul.vx are not supported
2303	* for EEW=64 in Zve64*. (Section 18.2)
2304	*/
2305	return opivx_check(s, a) &&
2306	(!has_ext(s, RVV) ? s->sew != MO_64 : true);
2307	}
2308
2309	GEN_OPIVV_TRANS(vsmul_vv, vsmul_vv_check)
2310	GEN_OPIVX_TRANS(vsmul_vx, vsmul_vx_check)
2311
2312	/* Vector Single-Width Scaling Shift Instructions */
2313	GEN_OPIVV_TRANS(vssrl_vv, opivv_check)
2314	GEN_OPIVV_TRANS(vssra_vv, opivv_check)
2315	GEN_OPIVX_TRANS(vssrl_vx, opivx_check)
2316	GEN_OPIVX_TRANS(vssra_vx, opivx_check)
2317	GEN_OPIVI_TRANS(vssrl_vi, IMM_TRUNC_SEW, vssrl_vx, opivx_check)
2318	GEN_OPIVI_TRANS(vssra_vi, IMM_TRUNC_SEW, vssra_vx, opivx_check)
2319
2320	/* Vector Narrowing Fixed-Point Clip Instructions */
2321	GEN_OPIWV_NARROW_TRANS(vnclipu_wv)
2322	GEN_OPIWV_NARROW_TRANS(vnclip_wv)
2323	GEN_OPIWX_NARROW_TRANS(vnclipu_wx)
2324	GEN_OPIWX_NARROW_TRANS(vnclip_wx)
2325	GEN_OPIWI_NARROW_TRANS(vnclipu_wi, IMM_ZX, vnclipu_wx)
2326	GEN_OPIWI_NARROW_TRANS(vnclip_wi, IMM_ZX, vnclip_wx)
2327
2328	/*
2329	*** Vector Float Point Arithmetic Instructions
2330	*/
2331
2332	/*
2333	* As RVF-only cpus always have values NaN-boxed to 64-bits,
2334	* RVF and RVD can be treated equally.
2335	* We don't have to deal with the cases of: SEW > FLEN.
2336	*
2337	* If SEW < FLEN, check whether input fp register is a valid
2338	* NaN-boxed value, in which case the least-significant SEW bits
2339	* of the f register are used, else the canonical NaN value is used.
2340	*/
2341	static void do_nanbox(DisasContext *s, TCGv_i64 out, TCGv_i64 in)
2342	{
2343	if (s->sew == MO_16) {
2344	if (s->altfmt) {
2345	gen_check_nanbox_h_bf16(out, in);
2346	} else {
2347	gen_check_nanbox_h(out, in);
2348	}
2349	} else if (s->sew == MO_32) {
2350	gen_check_nanbox_s(out, in);
2351	} else if (s->sew == MO_64) {
2352	tcg_gen_mov_i64(out, in);
2353	} else {
2354	g_assert_not_reached();
2355	}
2356	}
2357
2358	/*
2359	* Check altfmt & sew combinations when Zvfbfa extension is enabled.
2360	*/
2361	static bool vext_check_altfmt(DisasContext *s, int8_t valid_vsew)
2362	{
2363	if (s->cfg_ptr->ext_zvfbfa) {
2364	if (s->altfmt && (valid_vsew == -1 \|\| s->sew != valid_vsew)) {
2365	return false;
2366	}
2367	}
2368	return true;
2369	}
2370
2371	/* Vector Single-Width Floating-Point Add/Subtract Instructions */
2372
2373	/*
2374	* If the current SEW does not correspond to a supported IEEE floating-point
2375	* type, an illegal instruction exception is raised.
2376	*/
2377	static bool opfvv_check(DisasContext s, arg_rmrr a, int8_t valid_bfa_vsew)
2378	{
2379	return require_rvv(s) &&
2380	require_rvf(s) &&
2381	vext_check_isa_ill(s) &&
2382	vext_check_sss(s, a->rd, a->rs1, a->rs2, a->vm) &&
2383	vext_check_altfmt(s, valid_bfa_vsew);
2384	}
2385
2386	/* OPFVV without GVEC IR */
2387	#define GEN_OPFVV_TRANS(NAME, CHECK) \
2388	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2389	{ \
2390	if (CHECK(s, a, -1)) { \
2391	uint32_t data = 0; \
2392	static gen_helper_gvec_4_ptr * const fns[3] = { \
2393	gen_helper_##NAME##_h, \
2394	gen_helper_##NAME##_w, \
2395	gen_helper_##NAME##_d, \
2396	}; \
2397	gen_set_rm(s, RISCV_FRM_DYN); \
2398	\
2399	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2400	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2401	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2402	data = \
2403	FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\
2404	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2405	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
2406	vreg_ofs(s, a->rs1), \
2407	vreg_ofs(s, a->rs2), tcg_env, \
2408	s->cfg_ptr->vlenb, \
2409	s->cfg_ptr->vlenb, data, \
2410	fns[s->sew - 1]); \
2411	finalize_rvv_inst(s); \
2412	return true; \
2413	} \
2414	return false; \
2415	}
2416
2417	#define GEN_OPFVV_BFA_TRANS(NAME, CHECK, BFA_HELPER) \
2418	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2419	{ \
2420	if (CHECK(s, a, MO_16)) { \
2421	uint32_t data = 0; \
2422	static gen_helper_gvec_4_ptr * const fns[3] = { \
2423	gen_helper_##NAME##_h, \
2424	gen_helper_##NAME##_w, \
2425	gen_helper_##NAME##_d \
2426	}; \
2427	gen_set_rm(s, RISCV_FRM_DYN); \
2428	\
2429	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2430	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2431	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2432	data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); \
2433	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2434	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
2435	vreg_ofs(s, a->rs1), \
2436	vreg_ofs(s, a->rs2), tcg_env, \
2437	s->cfg_ptr->vlenb, \
2438	s->cfg_ptr->vlenb, data, \
2439	(s->altfmt ? gen_helper_##BFA_HELPER : \
2440	fns[s->sew - 1])); \
2441	tcg_gen_movi_i32(cpu_vstart, 0); \
2442	finalize_rvv_inst(s); \
2443	\
2444	return true; \
2445	} \
2446	return false; \
2447	}
2448
2449	GEN_OPFVV_BFA_TRANS(vfadd_vv, opfvv_check, vfadd_vv_h_bf16)
2450	GEN_OPFVV_BFA_TRANS(vfsub_vv, opfvv_check, vfsub_vv_h_bf16)
2451
2452	typedef void gen_helper_opfvf(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_ptr,
2453	TCGv_env, TCGv_i32);
2454
2455	static bool opfvf_trans(uint32_t vd, uint32_t rs1, uint32_t vs2,
2456	uint32_t data, gen_helper_opfvf fn, DisasContext s)
2457	{
2458	TCGv_ptr dest, src2, mask;
2459	TCGv_i32 desc;
2460	TCGv_i64 t1;
2461
2462	dest = tcg_temp_new_ptr();
2463	mask = tcg_temp_new_ptr();
2464	src2 = tcg_temp_new_ptr();
2465	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
2466	s->cfg_ptr->vlenb, data));
2467
2468	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vd));
2469	tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, vs2));
2470	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
2471
2472	/* NaN-box f[rs1] */
2473	t1 = tcg_temp_new_i64();
2474	do_nanbox(s, t1, cpu_fpr[rs1]);
2475
2476	fn(dest, mask, t1, src2, tcg_env, desc);
2477
2478	finalize_rvv_inst(s);
2479	return true;
2480	}
2481
2482	/*
2483	* If the current SEW does not correspond to a supported IEEE floating-point
2484	* type, an illegal instruction exception is raised
2485	*/
2486	static bool opfvf_check(DisasContext s, arg_rmrr a, int8_t valid_bfa_vsew)
2487	{
2488	return require_rvv(s) &&
2489	require_rvf(s) &&
2490	vext_check_isa_ill(s) &&
2491	vext_check_ss(s, a->rd, a->rs2, a->vm) &&
2492	vext_check_altfmt(s, valid_bfa_vsew);
2493	}
2494
2495	/* OPFVF without GVEC IR */
2496	#define GEN_OPFVF_TRANS(NAME, CHECK) \
2497	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2498	{ \
2499	if (CHECK(s, a, -1)) { \
2500	uint32_t data = 0; \
2501	static gen_helper_opfvf *const fns[3] = { \
2502	gen_helper_##NAME##_h, \
2503	gen_helper_##NAME##_w, \
2504	gen_helper_##NAME##_d, \
2505	}; \
2506	gen_set_rm(s, RISCV_FRM_DYN); \
2507	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2508	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2509	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2510	data = FIELD_DP32(data, VDATA, VTA_ALL_1S, \
2511	s->cfg_vta_all_1s); \
2512	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2513	return opfvf_trans(a->rd, a->rs1, a->rs2, data, \
2514	fns[s->sew - 1], s); \
2515	} \
2516	return false; \
2517	}
2518
2519	#define GEN_OPFVF_BFA_TRANS(NAME, CHECK, BFA_HELPER) \
2520	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2521	{ \
2522	if (CHECK(s, a, MO_16)) { \
2523	uint32_t data = 0; \
2524	static gen_helper_opfvf *const fns[3] = { \
2525	gen_helper_##NAME##_h, \
2526	gen_helper_##NAME##_w, \
2527	gen_helper_##NAME##_d, \
2528	}; \
2529	gen_set_rm(s, RISCV_FRM_DYN); \
2530	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2531	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2532	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2533	data = FIELD_DP32(data, VDATA, VTA_ALL_1S, \
2534	s->cfg_vta_all_1s); \
2535	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2536	return opfvf_trans(a->rd, a->rs1, a->rs2, data, \
2537	(s->altfmt ? gen_helper_##BFA_HELPER : \
2538	fns[s->sew - 1]), \
2539	s); \
2540	} \
2541	return false; \
2542	}
2543
2544	GEN_OPFVF_BFA_TRANS(vfadd_vf, opfvf_check, vfadd_vf_h_bf16)
2545	GEN_OPFVF_BFA_TRANS(vfsub_vf, opfvf_check, vfsub_vf_h_bf16)
2546	GEN_OPFVF_BFA_TRANS(vfrsub_vf, opfvf_check, vfrsub_vf_h_bf16)
2547
2548	/* Vector Widening Floating-Point Add/Subtract Instructions */
2549	static bool opfvv_widen_check(DisasContext s, arg_rmrr a,
2550	int8_t valid_bfa_vsew)
2551	{
2552	return require_rvv(s) &&
2553	require_rvf(s) &&
2554	require_scale_rvf(s) &&
2555	vext_check_isa_ill(s) &&
2556	vext_check_dss(s, a->rd, a->rs1, a->rs2, a->vm) &&
2557	vext_check_altfmt(s, valid_bfa_vsew);
2558	}
2559
2560	static bool opfvv_overwrite_widen_check(DisasContext s, arg_rmrr a,
2561	int8_t valid_bfa_vsew)
2562	{
2563	return opfvv_widen_check(s, a, valid_bfa_vsew) &&
2564	vext_check_input_eew(s, a->rd, s->sew + 1, a->rs1, s->sew, a->vm) &&
2565	vext_check_input_eew(s, a->rd, s->sew + 1, a->rs2, s->sew, a->vm);
2566	}
2567
2568	/* OPFVV with WIDEN */
2569	#define GEN_OPFVV_WIDEN_TRANS(NAME, CHECK) \
2570	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2571	{ \
2572	if (CHECK(s, a, -1)) { \
2573	uint32_t data = 0; \
2574	static gen_helper_gvec_4_ptr * const fns[2] = { \
2575	gen_helper_##NAME##_h, gen_helper_##NAME##_w, \
2576	}; \
2577	gen_set_rm(s, RISCV_FRM_DYN); \
2578	\
2579	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2580	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2581	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2582	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2583	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
2584	vreg_ofs(s, a->rs1), \
2585	vreg_ofs(s, a->rs2), tcg_env, \
2586	s->cfg_ptr->vlenb, \
2587	s->cfg_ptr->vlenb, data, \
2588	fns[s->sew - 1]); \
2589	finalize_rvv_inst(s); \
2590	return true; \
2591	} \
2592	return false; \
2593	}
2594
2595	#define GEN_OPFVV_WIDEN_BFA_TRANS(NAME, CHECK, BFA_HELPER) \
2596	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2597	{ \
2598	if (CHECK(s, a, MO_16)) { \
2599	uint32_t data = 0; \
2600	static gen_helper_gvec_4_ptr * const fns[2] = { \
2601	gen_helper_##NAME##_h, \
2602	gen_helper_##NAME##_w \
2603	}; \
2604	gen_set_rm(s, RISCV_FRM_DYN); \
2605	\
2606	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2607	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2608	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2609	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2610	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
2611	vreg_ofs(s, a->rs1), \
2612	vreg_ofs(s, a->rs2), tcg_env, \
2613	s->cfg_ptr->vlenb, \
2614	s->cfg_ptr->vlenb, data, \
2615	(s->altfmt ? gen_helper_##BFA_HELPER : \
2616	fns[s->sew - 1])); \
2617	finalize_rvv_inst(s); \
2618	return true; \
2619	} \
2620	return false; \
2621	}
2622
2623	GEN_OPFVV_WIDEN_BFA_TRANS(vfwadd_vv, opfvv_widen_check, vfwadd_vv_h_bf16)
2624	GEN_OPFVV_WIDEN_BFA_TRANS(vfwsub_vv, opfvv_widen_check, vfwsub_vv_h_bf16)
2625
2626	static bool opfvf_widen_check(DisasContext s, arg_rmrr a,
2627	int8_t valid_bfa_vsew)
2628	{
2629	return require_rvv(s) &&
2630	require_rvf(s) &&
2631	require_scale_rvf(s) &&
2632	vext_check_isa_ill(s) &&
2633	vext_check_ds(s, a->rd, a->rs2, a->vm) &&
2634	vext_check_altfmt(s, valid_bfa_vsew);
2635	}
2636
2637	static bool opfvf_overwrite_widen_check(DisasContext s, arg_rmrr a,
2638	int8_t valid_bfa_vsew)
2639	{
2640	return opfvf_widen_check(s, a, valid_bfa_vsew) &&
2641	vext_check_input_eew(s, a->rd, s->sew + 1, a->rs2, s->sew, a->vm);
2642	}
2643
2644	/* OPFVF with WIDEN */
2645	#define GEN_OPFVF_WIDEN_BFA_TRANS(NAME, CHECK, BFA_HELPER) \
2646	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2647	{ \
2648	if (CHECK(s, a, MO_16)) { \
2649	uint32_t data = 0; \
2650	static gen_helper_opfvf *const fns[2] = { \
2651	gen_helper_##NAME##_h, \
2652	gen_helper_##NAME##_w, \
2653	}; \
2654	gen_set_rm(s, RISCV_FRM_DYN); \
2655	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2656	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2657	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2658	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2659	return opfvf_trans(a->rd, a->rs1, a->rs2, data, \
2660	(s->altfmt ? gen_helper_##BFA_HELPER : \
2661	fns[s->sew - 1]), \
2662	s); \
2663	} \
2664	return false; \
2665	}
2666
2667	GEN_OPFVF_WIDEN_BFA_TRANS(vfwadd_vf, opfvf_widen_check, vfwadd_vf_h_bf16)
2668	GEN_OPFVF_WIDEN_BFA_TRANS(vfwsub_vf, opfvf_widen_check, vfwsub_vf_h_bf16)
2669
2670	static bool opfwv_widen_check(DisasContext s, arg_rmrr a,
2671	int8_t valid_bfa_vsew)
2672	{
2673	return require_rvv(s) &&
2674	require_rvf(s) &&
2675	require_scale_rvf(s) &&
2676	vext_check_isa_ill(s) &&
2677	vext_check_dds(s, a->rd, a->rs1, a->rs2, a->vm) &&
2678	vext_check_altfmt(s, valid_bfa_vsew);
2679	}
2680
2681	/* WIDEN OPFVV with WIDEN */
2682	#define GEN_OPFWV_WIDEN_BFA_TRANS(NAME) \
2683	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2684	{ \
2685	if (opfwv_widen_check(s, a, MO_16)) { \
2686	uint32_t data = 0; \
2687	static gen_helper_gvec_4_ptr * const fns[2] = { \
2688	gen_helper_##NAME##_h, \
2689	gen_helper_##NAME##_w \
2690	}; \
2691	gen_set_rm(s, RISCV_FRM_DYN); \
2692	\
2693	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2694	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2695	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2696	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2697	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
2698	vreg_ofs(s, a->rs1), \
2699	vreg_ofs(s, a->rs2), tcg_env, \
2700	s->cfg_ptr->vlenb, \
2701	s->cfg_ptr->vlenb, data, \
2702	(s->altfmt ? gen_helper_##NAME##_h_bf16 : \
2703	fns[s->sew - 1])); \
2704	finalize_rvv_inst(s); \
2705	return true; \
2706	} \
2707	return false; \
2708	}
2709
2710	GEN_OPFWV_WIDEN_BFA_TRANS(vfwadd_wv)
2711	GEN_OPFWV_WIDEN_BFA_TRANS(vfwsub_wv)
2712
2713	static bool opfwf_widen_check(DisasContext s, arg_rmrr a,
2714	int8_t valid_bfa_vsew)
2715	{
2716	return require_rvv(s) &&
2717	require_rvf(s) &&
2718	require_scale_rvf(s) &&
2719	vext_check_isa_ill(s) &&
2720	vext_check_dd(s, a->rd, a->rs2, a->vm) &&
2721	vext_check_altfmt(s, valid_bfa_vsew);
2722	}
2723
2724	/* WIDEN OPFVF with WIDEN */
2725	#define GEN_OPFWF_WIDEN_BFA_TRANS(NAME) \
2726	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
2727	{ \
2728	if (opfwf_widen_check(s, a, MO_16)) { \
2729	uint32_t data = 0; \
2730	static gen_helper_opfvf *const fns[2] = { \
2731	gen_helper_##NAME##_h, \
2732	gen_helper_##NAME##_w \
2733	}; \
2734	gen_set_rm(s, RISCV_FRM_DYN); \
2735	data = FIELD_DP32(data, VDATA, VM, a->vm); \
2736	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
2737	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
2738	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
2739	return opfvf_trans(a->rd, a->rs1, a->rs2, data, \
2740	(s->altfmt ? gen_helper_##NAME##_h_bf16 : \
2741	fns[s->sew - 1]), \
2742	s); \
2743	} \
2744	return false; \
2745	}
2746
2747	GEN_OPFWF_WIDEN_BFA_TRANS(vfwadd_wf)
2748	GEN_OPFWF_WIDEN_BFA_TRANS(vfwsub_wf)
2749
2750	/* Vector Single-Width Floating-Point Multiply/Divide Instructions */
2751	GEN_OPFVV_BFA_TRANS(vfmul_vv, opfvv_check, vfmul_vv_h_bf16)
2752	GEN_OPFVV_TRANS(vfdiv_vv, opfvv_check)
2753	GEN_OPFVF_BFA_TRANS(vfmul_vf, opfvf_check, vfmul_vf_h_bf16)
2754	GEN_OPFVF_TRANS(vfdiv_vf, opfvf_check)
2755	GEN_OPFVF_TRANS(vfrdiv_vf, opfvf_check)
2756
2757	/* Vector Widening Floating-Point Multiply */
2758	GEN_OPFVV_WIDEN_BFA_TRANS(vfwmul_vv, opfvv_widen_check, vfwmul_vv_h_bf16)
2759	GEN_OPFVF_WIDEN_BFA_TRANS(vfwmul_vf, opfvf_widen_check, vfwmul_vf_h_bf16)
2760
2761	/* Vector Single-Width Floating-Point Fused Multiply-Add Instructions */
2762	GEN_OPFVV_BFA_TRANS(vfmacc_vv, opfvv_check, vfmacc_vv_h_bf16)
2763	GEN_OPFVV_BFA_TRANS(vfnmacc_vv, opfvv_check, vfnmacc_vv_h_bf16)
2764	GEN_OPFVV_BFA_TRANS(vfmsac_vv, opfvv_check, vfmsac_vv_h_bf16)
2765	GEN_OPFVV_BFA_TRANS(vfnmsac_vv, opfvv_check, vfnmsac_vv_h_bf16)
2766	GEN_OPFVV_BFA_TRANS(vfmadd_vv, opfvv_check, vfmadd_vv_h_bf16)
2767	GEN_OPFVV_BFA_TRANS(vfnmadd_vv, opfvv_check, vfnmadd_vv_h_bf16)
2768	GEN_OPFVV_BFA_TRANS(vfmsub_vv, opfvv_check, vfmsub_vv_h_bf16)
2769	GEN_OPFVV_BFA_TRANS(vfnmsub_vv, opfvv_check, vfnmsub_vv_h_bf16)
2770	GEN_OPFVF_BFA_TRANS(vfmacc_vf, opfvf_check, vfmacc_vf_h_bf16)
2771	GEN_OPFVF_BFA_TRANS(vfnmacc_vf, opfvf_check, vfnmacc_vf_h_bf16)
2772	GEN_OPFVF_BFA_TRANS(vfmsac_vf, opfvf_check, vfmsac_vf_h_bf16)
2773	GEN_OPFVF_BFA_TRANS(vfnmsac_vf, opfvf_check, vfnmsac_vf_h_bf16)
2774	GEN_OPFVF_BFA_TRANS(vfmadd_vf, opfvf_check, vfmadd_vf_h_bf16)
2775	GEN_OPFVF_BFA_TRANS(vfnmadd_vf, opfvf_check, vfnmadd_vf_h_bf16)
2776	GEN_OPFVF_BFA_TRANS(vfmsub_vf, opfvf_check, vfmsub_vf_h_bf16)
2777	GEN_OPFVF_BFA_TRANS(vfnmsub_vf, opfvf_check, vfnmsub_vf_h_bf16)
2778
2779	/* Vector Widening Floating-Point Fused Multiply-Add Instructions */
2780	GEN_OPFVV_WIDEN_BFA_TRANS(vfwmacc_vv, opfvv_overwrite_widen_check,
2781	vfwmaccbf16_vv)
2782	GEN_OPFVV_WIDEN_BFA_TRANS(vfwnmacc_vv, opfvv_overwrite_widen_check,
2783	vfwnmacc_vv_h_bf16)
2784	GEN_OPFVV_WIDEN_BFA_TRANS(vfwmsac_vv, opfvv_overwrite_widen_check,
2785	vfwmsac_vv_h_bf16)
2786	GEN_OPFVV_WIDEN_BFA_TRANS(vfwnmsac_vv, opfvv_overwrite_widen_check,
2787	vfwnmsac_vv_h_bf16)
2788	GEN_OPFVF_WIDEN_BFA_TRANS(vfwmacc_vf, opfvf_overwrite_widen_check,
2789	vfwmaccbf16_vf)
2790	GEN_OPFVF_WIDEN_BFA_TRANS(vfwnmacc_vf, opfvf_overwrite_widen_check,
2791	vfwnmacc_vf_h_bf16)
2792	GEN_OPFVF_WIDEN_BFA_TRANS(vfwmsac_vf, opfvf_overwrite_widen_check,
2793	vfwmsac_vf_h_bf16)
2794	GEN_OPFVF_WIDEN_BFA_TRANS(vfwnmsac_vf, opfvf_overwrite_widen_check,
2795	vfwnmsac_vf_h_bf16)
2796
2797	/* Vector Floating-Point Square-Root Instruction */
2798
2799	/*
2800	* If the current SEW does not correspond to a supported IEEE floating-point
2801	* type, an illegal instruction exception is raised
2802	*/
2803	static bool opfv_check(DisasContext s, arg_rmr a, int8_t valid_bfa_vsew)
2804	{
2805	return require_rvv(s) &&
2806	require_rvf(s) &&
2807	vext_check_isa_ill(s) &&
2808	/* OPFV instructions ignore vs1 check */
2809	vext_check_ss(s, a->rd, a->rs2, a->vm) &&
2810	vext_check_altfmt(s, valid_bfa_vsew);
2811	}
2812
2813	static bool do_opfv(DisasContext s, arg_rmr a,
2814	gen_helper_gvec_3_ptr *fn,
2815	bool (checkfn)(DisasContext , arg_rmr *, int8_t),
2816	int rm,
2817	int8_t valid_bfa_vsew)
2818	{
2819	if (checkfn(s, a, valid_bfa_vsew)) {
2820	uint32_t data = 0;
2821	gen_set_rm_chkfrm(s, rm);
2822
2823	data = FIELD_DP32(data, VDATA, VM, a->vm);
2824	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
2825	data = FIELD_DP32(data, VDATA, VTA, s->vta);
2826	data = FIELD_DP32(data, VDATA, VMA, s->vma);
2827	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
2828	vreg_ofs(s, a->rs2), tcg_env,
2829	s->cfg_ptr->vlenb,
2830	s->cfg_ptr->vlenb, data, fn);
2831	finalize_rvv_inst(s);
2832	return true;
2833	}
2834	return false;
2835	}
2836
2837	#define GEN_OPFV_TRANS(NAME, CHECK, FRM) \
2838	static bool trans_##NAME(DisasContext s, arg_rmr a) \
2839	{ \
2840	static gen_helper_gvec_3_ptr * const fns[3] = { \
2841	gen_helper_##NAME##_h, \
2842	gen_helper_##NAME##_w, \
2843	gen_helper_##NAME##_d \
2844	}; \
2845	return do_opfv(s, a, fns[s->sew - 1], CHECK, FRM, -1); \
2846	}
2847
2848	#define GEN_OPFV_BFA_TRANS(NAME, CHECK, FRM) \
2849	static bool trans_##NAME(DisasContext s, arg_rmr a) \
2850	{ \
2851	static gen_helper_gvec_3_ptr * const fns[3] = { \
2852	gen_helper_##NAME##_h, \
2853	gen_helper_##NAME##_w, \
2854	gen_helper_##NAME##_d \
2855	}; \
2856	return do_opfv(s, a, \
2857	(s->altfmt ? gen_helper_##NAME##_h_bf16 : \
2858	fns[s->sew - 1]), \
2859	CHECK, FRM, MO_16); \
2860	}
2861
2862	GEN_OPFV_TRANS(vfsqrt_v, opfv_check, RISCV_FRM_DYN)
2863	GEN_OPFV_BFA_TRANS(vfrsqrt7_v, opfv_check, RISCV_FRM_DYN)
2864	GEN_OPFV_BFA_TRANS(vfrec7_v, opfv_check, RISCV_FRM_DYN)
2865
2866	/* Vector Floating-Point MIN/MAX Instructions */
2867	GEN_OPFVV_BFA_TRANS(vfmin_vv, opfvv_check, vfmin_vv_h_bf16)
2868	GEN_OPFVV_BFA_TRANS(vfmax_vv, opfvv_check, vfmax_vv_h_bf16)
2869	GEN_OPFVF_BFA_TRANS(vfmin_vf, opfvf_check, vfmin_vf_h_bf16)
2870	GEN_OPFVF_BFA_TRANS(vfmax_vf, opfvf_check, vfmax_vf_h_bf16)
2871
2872	/* Vector Floating-Point Sign-Injection Instructions */
2873	GEN_OPFVV_BFA_TRANS(vfsgnj_vv, opfvv_check, vfsgnj_vv_h)
2874	GEN_OPFVV_BFA_TRANS(vfsgnjn_vv, opfvv_check, vfsgnjn_vv_h)
2875	GEN_OPFVV_BFA_TRANS(vfsgnjx_vv, opfvv_check, vfsgnjx_vv_h)
2876	GEN_OPFVF_BFA_TRANS(vfsgnj_vf, opfvf_check, vfsgnj_vf_h)
2877	GEN_OPFVF_BFA_TRANS(vfsgnjn_vf, opfvf_check, vfsgnjn_vf_h)
2878	GEN_OPFVF_BFA_TRANS(vfsgnjx_vf, opfvf_check, vfsgnjx_vf_h)
2879
2880	/* Vector Floating-Point Compare Instructions */
2881	static bool opfvv_cmp_check(DisasContext s, arg_rmrr a,
2882	int8_t valid_bfa_vsew)
2883	{
2884	return require_rvv(s) &&
2885	require_rvf(s) &&
2886	vext_check_isa_ill(s) &&
2887	vext_check_mss(s, a->rd, a->rs1, a->rs2) &&
2888	vext_check_altfmt(s, valid_bfa_vsew);
2889	}
2890
2891	GEN_OPFVV_BFA_TRANS(vmfeq_vv, opfvv_cmp_check, vmfeq_vv_h_bf16)
2892	GEN_OPFVV_BFA_TRANS(vmfne_vv, opfvv_cmp_check, vmfne_vv_h_bf16)
2893	GEN_OPFVV_BFA_TRANS(vmflt_vv, opfvv_cmp_check, vmflt_vv_h_bf16)
2894	GEN_OPFVV_BFA_TRANS(vmfle_vv, opfvv_cmp_check, vmfle_vv_h_bf16)
2895
2896	static bool opfvf_cmp_check(DisasContext s, arg_rmrr a,
2897	int8_t valid_bfa_vsew)
2898	{
2899	return require_rvv(s) &&
2900	require_rvf(s) &&
2901	vext_check_isa_ill(s) &&
2902	vext_check_ms(s, a->rd, a->rs2) &&
2903	vext_check_altfmt(s, valid_bfa_vsew);
2904	}
2905
2906	GEN_OPFVF_BFA_TRANS(vmfeq_vf, opfvf_cmp_check, vmfeq_vf_h_bf16)
2907	GEN_OPFVF_BFA_TRANS(vmfne_vf, opfvf_cmp_check, vmfne_vf_h_bf16)
2908	GEN_OPFVF_BFA_TRANS(vmflt_vf, opfvf_cmp_check, vmflt_vf_h_bf16)
2909	GEN_OPFVF_BFA_TRANS(vmfle_vf, opfvf_cmp_check, vmfle_vf_h_bf16)
2910	GEN_OPFVF_BFA_TRANS(vmfgt_vf, opfvf_cmp_check, vmfgt_vf_h_bf16)
2911	GEN_OPFVF_BFA_TRANS(vmfge_vf, opfvf_cmp_check, vmfge_vf_h_bf16)
2912
2913	/* Vector Floating-Point Classify Instruction */
2914	GEN_OPFV_BFA_TRANS(vfclass_v, opfv_check, RISCV_FRM_DYN)
2915
2916	/* Vector Floating-Point Merge Instruction */
2917	GEN_OPFVF_BFA_TRANS(vfmerge_vfm, opfvf_check, vfmerge_vfm_h)
2918
2919	static bool trans_vfmv_v_f(DisasContext s, arg_vfmv_v_f a)
2920	{
2921	if (require_rvv(s) &&
2922	require_rvf(s) &&
2923	vext_check_isa_ill(s) &&
2924	require_align(a->rd, s->lmul) &&
2925	vext_check_altfmt(s, MO_16)) {
2926	gen_set_rm(s, RISCV_FRM_DYN);
2927
2928	TCGv_i64 t1;
2929
2930	if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
2931	t1 = tcg_temp_new_i64();
2932	/* NaN-box f[rs1] */
2933	do_nanbox(s, t1, cpu_fpr[a->rs1]);
2934
2935	tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd),
2936	MAXSZ(s), MAXSZ(s), t1);
2937	} else {
2938	TCGv_ptr dest;
2939	TCGv_i32 desc;
2940	uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul);
2941	data = FIELD_DP32(data, VDATA, VTA, s->vta);
2942	data = FIELD_DP32(data, VDATA, VMA, s->vma);
2943	static gen_helper_vmv_vx * const fns[3] = {
2944	gen_helper_vmv_v_x_h,
2945	gen_helper_vmv_v_x_w,
2946	gen_helper_vmv_v_x_d
2947	};
2948
2949	t1 = tcg_temp_new_i64();
2950	/* NaN-box f[rs1] */
2951	do_nanbox(s, t1, cpu_fpr[a->rs1]);
2952
2953	dest = tcg_temp_new_ptr();
2954	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
2955	s->cfg_ptr->vlenb, data));
2956	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd));
2957
2958	fns[s->sew - 1](dest, t1, tcg_env, desc);
2959	}
2960	finalize_rvv_inst(s);
2961	return true;
2962	}
2963	return false;
2964	}
2965
2966	/* Single-Width Floating-Point/Integer Type-Convert Instructions */
2967	#define GEN_OPFV_CVT_TRANS(NAME, HELPER, FRM) \
2968	static bool trans_##NAME(DisasContext s, arg_rmr a) \
2969	{ \
2970	static gen_helper_gvec_3_ptr * const fns[3] = { \
2971	gen_helper_##HELPER##_h, \
2972	gen_helper_##HELPER##_w, \
2973	gen_helper_##HELPER##_d \
2974	}; \
2975	return do_opfv(s, a, fns[s->sew - 1], opfv_check, FRM, -1); \
2976	}
2977
2978	GEN_OPFV_CVT_TRANS(vfcvt_xu_f_v, vfcvt_xu_f_v, RISCV_FRM_DYN)
2979	GEN_OPFV_CVT_TRANS(vfcvt_x_f_v, vfcvt_x_f_v, RISCV_FRM_DYN)
2980	GEN_OPFV_CVT_TRANS(vfcvt_f_xu_v, vfcvt_f_xu_v, RISCV_FRM_DYN)
2981	GEN_OPFV_CVT_TRANS(vfcvt_f_x_v, vfcvt_f_x_v, RISCV_FRM_DYN)
2982	/* Reuse the helper functions from vfcvt.xu.f.v and vfcvt.x.f.v */
2983	GEN_OPFV_CVT_TRANS(vfcvt_rtz_xu_f_v, vfcvt_xu_f_v, RISCV_FRM_RTZ)
2984	GEN_OPFV_CVT_TRANS(vfcvt_rtz_x_f_v, vfcvt_x_f_v, RISCV_FRM_RTZ)
2985
2986	/* Widening Floating-Point/Integer Type-Convert Instructions */
2987
2988	/*
2989	* If the current SEW does not correspond to a supported IEEE floating-point
2990	* type, an illegal instruction exception is raised
2991	*/
2992	static bool opfv_widen_check(DisasContext s, arg_rmr a)
2993	{
2994	return require_rvv(s) &&
2995	vext_check_isa_ill(s) &&
2996	vext_check_ds(s, a->rd, a->rs2, a->vm);
2997	}
2998
2999	static bool opxfv_widen_check(DisasContext s, arg_rmr a,
3000	int8_t valid_bfa_vsew)
3001	{
3002	return opfv_widen_check(s, a) &&
3003	require_rvf(s) &&
3004	vext_check_altfmt(s, valid_bfa_vsew);
3005	}
3006
3007	static bool opffv_widen_check(DisasContext s, arg_rmr a,
3008	int8_t valid_bfa_vsew)
3009	{
3010	return opfv_widen_check(s, a) &&
3011	require_rvfmin(s) &&
3012	require_scale_rvfmin(s) &&
3013	vext_check_altfmt(s, valid_bfa_vsew);
3014	}
3015
3016	#define GEN_OPFV_WIDEN_TRANS(NAME, CHECK, HELPER, FRM) \
3017	static bool trans_##NAME(DisasContext s, arg_rmr a) \
3018	{ \
3019	if (CHECK(s, a, -1)) { \
3020	uint32_t data = 0; \
3021	static gen_helper_gvec_3_ptr * const fns[2] = { \
3022	gen_helper_##HELPER##_h, \
3023	gen_helper_##HELPER##_w, \
3024	}; \
3025	gen_set_rm_chkfrm(s, FRM); \
3026	\
3027	data = FIELD_DP32(data, VDATA, VM, a->vm); \
3028	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3029	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
3030	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
3031	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
3032	vreg_ofs(s, a->rs2), tcg_env, \
3033	s->cfg_ptr->vlenb, \
3034	s->cfg_ptr->vlenb, data, \
3035	fns[s->sew - 1]); \
3036	finalize_rvv_inst(s); \
3037	return true; \
3038	} \
3039	return false; \
3040	}
3041
3042	#define GEN_OPFV_WIDEN_BFA_TRANS(NAME, CHECK, HELPER, FRM, BFA_HELPER) \
3043	static bool trans_##NAME(DisasContext s, arg_rmr a) \
3044	{ \
3045	if (CHECK(s, a, MO_16)) { \
3046	uint32_t data = 0; \
3047	static gen_helper_gvec_3_ptr * const fns[2] = { \
3048	gen_helper_##HELPER##_h, \
3049	gen_helper_##HELPER##_w, \
3050	}; \
3051	gen_set_rm_chkfrm(s, FRM); \
3052	\
3053	data = FIELD_DP32(data, VDATA, VM, a->vm); \
3054	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3055	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
3056	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
3057	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
3058	vreg_ofs(s, a->rs2), tcg_env, \
3059	s->cfg_ptr->vlenb, \
3060	s->cfg_ptr->vlenb, data, \
3061	(s->altfmt ? gen_helper_##BFA_HELPER : \
3062	fns[s->sew - 1])); \
3063	finalize_rvv_inst(s); \
3064	return true; \
3065	} \
3066	return false; \
3067	}
3068
3069	GEN_OPFV_WIDEN_TRANS(vfwcvt_xu_f_v, opxfv_widen_check, vfwcvt_xu_f_v,
3070	RISCV_FRM_DYN)
3071	GEN_OPFV_WIDEN_TRANS(vfwcvt_x_f_v, opxfv_widen_check, vfwcvt_x_f_v,
3072	RISCV_FRM_DYN)
3073	GEN_OPFV_WIDEN_BFA_TRANS(vfwcvt_f_f_v, opffv_widen_check, vfwcvt_f_f_v,
3074	RISCV_FRM_DYN, vfwcvtbf16_f_f_v)
3075	/* Reuse the helper functions from vfwcvt.xu.f.v and vfwcvt.x.f.v */
3076	GEN_OPFV_WIDEN_TRANS(vfwcvt_rtz_xu_f_v, opxfv_widen_check, vfwcvt_xu_f_v,
3077	RISCV_FRM_RTZ)
3078	GEN_OPFV_WIDEN_TRANS(vfwcvt_rtz_x_f_v, opxfv_widen_check, vfwcvt_x_f_v,
3079	RISCV_FRM_RTZ)
3080
3081	static bool opfxv_widen_check(DisasContext s, arg_rmr a,
3082	int8_t valid_bfa_vsew)
3083	{
3084	return require_rvv(s) &&
3085	require_scale_rvf(s) &&
3086	vext_check_isa_ill(s) &&
3087	/* OPFV widening instructions ignore vs1 check */
3088	vext_check_ds(s, a->rd, a->rs2, a->vm) &&
3089	vext_check_altfmt(s, valid_bfa_vsew);
3090	}
3091
3092	#define GEN_OPFXV_WIDEN_BFA_TRANS(NAME) \
3093	static bool trans_##NAME(DisasContext s, arg_rmr a) \
3094	{ \
3095	if (opfxv_widen_check(s, a, MO_8)) { \
3096	uint32_t data = 0; \
3097	static gen_helper_gvec_3_ptr * const fns[3] = { \
3098	gen_helper_##NAME##_b, \
3099	gen_helper_##NAME##_h, \
3100	gen_helper_##NAME##_w \
3101	}; \
3102	gen_set_rm(s, RISCV_FRM_DYN); \
3103	\
3104	data = FIELD_DP32(data, VDATA, VM, a->vm); \
3105	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3106	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
3107	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
3108	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
3109	vreg_ofs(s, a->rs2), tcg_env, \
3110	s->cfg_ptr->vlenb, \
3111	s->cfg_ptr->vlenb, data, \
3112	(s->altfmt ? gen_helper_##NAME##_b_bf16 : \
3113	fns[s->sew])); \
3114	finalize_rvv_inst(s); \
3115	return true; \
3116	} \
3117	return false; \
3118	}
3119
3120	GEN_OPFXV_WIDEN_BFA_TRANS(vfwcvt_f_xu_v)
3121	GEN_OPFXV_WIDEN_BFA_TRANS(vfwcvt_f_x_v)
3122
3123	/* Narrowing Floating-Point/Integer Type-Convert Instructions */
3124
3125	/*
3126	* If the current SEW does not correspond to a supported IEEE floating-point
3127	* type, an illegal instruction exception is raised
3128	*/
3129	static bool opfv_narrow_check(DisasContext s, arg_rmr a)
3130	{
3131	return require_rvv(s) &&
3132	vext_check_isa_ill(s) &&
3133	/* OPFV narrowing instructions ignore vs1 check */
3134	vext_check_sd(s, a->rd, a->rs2, a->vm);
3135	}
3136
3137	static bool opfxv_narrow_check(DisasContext s, arg_rmr a,
3138	int8_t valid_bfa_vsew)
3139	{
3140	return opfv_narrow_check(s, a) &&
3141	require_rvf(s) &&
3142	(s->sew != MO_64) &&
3143	vext_check_altfmt(s, valid_bfa_vsew);
3144	}
3145
3146	static bool opffv_narrow_check(DisasContext s, arg_rmr a,
3147	int8_t valid_bfa_vsew)
3148	{
3149	return opfv_narrow_check(s, a) &&
3150	require_rvfmin(s) &&
3151	require_scale_rvfmin(s) &&
3152	vext_check_altfmt(s, valid_bfa_vsew);
3153	}
3154
3155	static bool opffv_rod_narrow_check(DisasContext s, arg_rmr a,
3156	int8_t valid_bfa_vsew)
3157	{
3158	return opfv_narrow_check(s, a) &&
3159	require_rvf(s) &&
3160	require_scale_rvf(s) &&
3161	vext_check_altfmt(s, valid_bfa_vsew);
3162	}
3163
3164	#define GEN_OPFV_NARROW_TRANS(NAME, CHECK, HELPER, FRM) \
3165	static bool trans_##NAME(DisasContext s, arg_rmr a) \
3166	{ \
3167	if (CHECK(s, a, -1)) { \
3168	uint32_t data = 0; \
3169	static gen_helper_gvec_3_ptr * const fns[2] = { \
3170	gen_helper_##HELPER##_h, \
3171	gen_helper_##HELPER##_w, \
3172	}; \
3173	gen_set_rm_chkfrm(s, FRM); \
3174	\
3175	data = FIELD_DP32(data, VDATA, VM, a->vm); \
3176	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3177	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
3178	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
3179	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
3180	vreg_ofs(s, a->rs2), tcg_env, \
3181	s->cfg_ptr->vlenb, \
3182	s->cfg_ptr->vlenb, data, \
3183	fns[s->sew - 1]); \
3184	finalize_rvv_inst(s); \
3185	return true; \
3186	} \
3187	return false; \
3188	}
3189
3190	#define GEN_OPFV_NARROW_BFA_TRANS(NAME, CHECK, HELPER, FRM, BFA_HELPER) \
3191	static bool trans_##NAME(DisasContext s, arg_rmr a) \
3192	{ \
3193	if (CHECK(s, a, MO_16)) { \
3194	uint32_t data = 0; \
3195	static gen_helper_gvec_3_ptr * const fns[2] = { \
3196	gen_helper_##HELPER##_h, \
3197	gen_helper_##HELPER##_w, \
3198	}; \
3199	gen_set_rm_chkfrm(s, FRM); \
3200	\
3201	data = FIELD_DP32(data, VDATA, VM, a->vm); \
3202	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3203	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
3204	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
3205	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
3206	vreg_ofs(s, a->rs2), tcg_env, \
3207	s->cfg_ptr->vlenb, \
3208	s->cfg_ptr->vlenb, data, \
3209	(s->altfmt ? gen_helper_##BFA_HELPER : \
3210	fns[s->sew - 1])); \
3211	finalize_rvv_inst(s); \
3212	return true; \
3213	} \
3214	return false; \
3215	}
3216
3217	GEN_OPFV_NARROW_TRANS(vfncvt_f_xu_w, opfxv_narrow_check, vfncvt_f_xu_w,
3218	RISCV_FRM_DYN)
3219	GEN_OPFV_NARROW_TRANS(vfncvt_f_x_w, opfxv_narrow_check, vfncvt_f_x_w,
3220	RISCV_FRM_DYN)
3221	GEN_OPFV_NARROW_BFA_TRANS(vfncvt_f_f_w, opffv_narrow_check, vfncvt_f_f_w,
3222	RISCV_FRM_DYN, vfncvtbf16_f_f_w)
3223	/* Reuse the helper function from vfncvt.f.f.w */
3224	GEN_OPFV_NARROW_BFA_TRANS(vfncvt_rod_f_f_w, opffv_rod_narrow_check,
3225	vfncvt_f_f_w, RISCV_FRM_ROD, vfncvtbf16_f_f_w)
3226
3227	static bool opxfv_narrow_check(DisasContext s, arg_rmr a,
3228	int8_t valid_bfa_vsew)
3229	{
3230	return require_rvv(s) &&
3231	require_scale_rvf(s) &&
3232	vext_check_isa_ill(s) &&
3233	/* OPFV narrowing instructions ignore vs1 check */
3234	vext_check_sd(s, a->rd, a->rs2, a->vm) &&
3235	vext_check_altfmt(s, valid_bfa_vsew);
3236	}
3237
3238	#define GEN_OPXFV_NARROW_BFA_TRANS(NAME, HELPER, FRM) \
3239	static bool trans_##NAME(DisasContext s, arg_rmr a) \
3240	{ \
3241	if (opxfv_narrow_check(s, a, MO_8)) { \
3242	uint32_t data = 0; \
3243	static gen_helper_gvec_3_ptr * const fns[3] = { \
3244	gen_helper_##HELPER##_b, \
3245	gen_helper_##HELPER##_h, \
3246	gen_helper_##HELPER##_w \
3247	}; \
3248	gen_set_rm_chkfrm(s, FRM); \
3249	\
3250	data = FIELD_DP32(data, VDATA, VM, a->vm); \
3251	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3252	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
3253	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
3254	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
3255	vreg_ofs(s, a->rs2), tcg_env, \
3256	s->cfg_ptr->vlenb, \
3257	s->cfg_ptr->vlenb, data, \
3258	(s->altfmt ? gen_helper_##HELPER##_b_bf16 : \
3259	fns[s->sew])); \
3260	finalize_rvv_inst(s); \
3261	return true; \
3262	} \
3263	return false; \
3264	}
3265
3266	GEN_OPXFV_NARROW_BFA_TRANS(vfncvt_xu_f_w, vfncvt_xu_f_w, RISCV_FRM_DYN)
3267	GEN_OPXFV_NARROW_BFA_TRANS(vfncvt_x_f_w, vfncvt_x_f_w, RISCV_FRM_DYN)
3268	/* Reuse the helper functions from vfncvt.xu.f.w and vfncvt.x.f.w */
3269	GEN_OPXFV_NARROW_BFA_TRANS(vfncvt_rtz_xu_f_w, vfncvt_xu_f_w, RISCV_FRM_RTZ)
3270	GEN_OPXFV_NARROW_BFA_TRANS(vfncvt_rtz_x_f_w, vfncvt_x_f_w, RISCV_FRM_RTZ)
3271
3272	/*
3273	*** Vector Reduction Operations
3274	*/
3275	/* Vector Single-Width Integer Reduction Instructions */
3276	static bool reduction_check(DisasContext s, arg_rmrr a)
3277	{
3278	return require_rvv(s) &&
3279	vext_check_isa_ill(s) &&
3280	require_vm(a->vm, a->rs1) &&
3281	require_vm(a->vm, a->rs2) &&
3282	vext_check_reduction(s, a->rs2);
3283	}
3284
3285	GEN_OPIVV_TRANS(vredsum_vs, reduction_check)
3286	GEN_OPIVV_TRANS(vredmaxu_vs, reduction_check)
3287	GEN_OPIVV_TRANS(vredmax_vs, reduction_check)
3288	GEN_OPIVV_TRANS(vredminu_vs, reduction_check)
3289	GEN_OPIVV_TRANS(vredmin_vs, reduction_check)
3290	GEN_OPIVV_TRANS(vredand_vs, reduction_check)
3291	GEN_OPIVV_TRANS(vredor_vs, reduction_check)
3292	GEN_OPIVV_TRANS(vredxor_vs, reduction_check)
3293
3294	/* Vector Widening Integer Reduction Instructions */
3295	static bool reduction_widen_check(DisasContext s, arg_rmrr a)
3296	{
3297	return reduction_check(s, a) && (s->sew < MO_64) &&
3298	!is_overlapped(a->rs1, 1, a->rs2, 1 << MAX(s->lmul, 0)) &&
3299	((s->sew + 1) <= (s->cfg_ptr->elen >> 4));
3300	}
3301
3302	GEN_OPIVV_WIDEN_TRANS(vwredsum_vs, reduction_widen_check)
3303	GEN_OPIVV_WIDEN_TRANS(vwredsumu_vs, reduction_widen_check)
3304
3305	/* Vector Single-Width Floating-Point Reduction Instructions */
3306	static bool freduction_check(DisasContext s, arg_rmrr a,
3307	int8_t valid_bfa_vsew)
3308	{
3309	return reduction_check(s, a) &&
3310	require_rvf(s) &&
3311	vext_check_altfmt(s, valid_bfa_vsew);
3312	}
3313
3314	GEN_OPFVV_TRANS(vfredusum_vs, freduction_check)
3315	GEN_OPFVV_TRANS(vfredosum_vs, freduction_check)
3316	GEN_OPFVV_TRANS(vfredmax_vs, freduction_check)
3317	GEN_OPFVV_TRANS(vfredmin_vs, freduction_check)
3318
3319	/* Vector Widening Floating-Point Reduction Instructions */
3320	static bool freduction_widen_check(DisasContext s, arg_rmrr a,
3321	int8_t valid_bfa_vsew)
3322	{
3323	return reduction_widen_check(s, a) &&
3324	require_rvf(s) &&
3325	require_scale_rvf(s) &&
3326	vext_check_altfmt(s, valid_bfa_vsew);
3327	}
3328
3329	GEN_OPFVV_WIDEN_TRANS(vfwredusum_vs, freduction_widen_check)
3330	GEN_OPFVV_WIDEN_TRANS(vfwredosum_vs, freduction_widen_check)
3331
3332	/*
3333	*** Vector Mask Operations
3334	*/
3335
3336	/* Vector Mask-Register Logical Instructions */
3337	#define GEN_MM_TRANS(NAME) \
3338	static bool trans_##NAME(DisasContext s, arg_r a) \
3339	{ \
3340	if (require_rvv(s) && \
3341	vext_check_isa_ill(s)) { \
3342	uint32_t data = 0; \
3343	gen_helper_gvec_4_ptr *fn = gen_helper_##NAME; \
3344	\
3345	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3346	data = \
3347	FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\
3348	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \
3349	vreg_ofs(s, a->rs1), \
3350	vreg_ofs(s, a->rs2), tcg_env, \
3351	s->cfg_ptr->vlenb, \
3352	s->cfg_ptr->vlenb, data, fn); \
3353	finalize_rvv_inst(s); \
3354	return true; \
3355	} \
3356	return false; \
3357	}
3358
3359	GEN_MM_TRANS(vmand_mm)
3360	GEN_MM_TRANS(vmnand_mm)
3361	GEN_MM_TRANS(vmandn_mm)
3362	GEN_MM_TRANS(vmxor_mm)
3363	GEN_MM_TRANS(vmor_mm)
3364	GEN_MM_TRANS(vmnor_mm)
3365	GEN_MM_TRANS(vmorn_mm)
3366	GEN_MM_TRANS(vmxnor_mm)
3367
3368	/* Vector count population in mask vcpop */
3369	static bool trans_vcpop_m(DisasContext s, arg_rmr a)
3370	{
3371	if (require_rvv(s) &&
3372	vext_check_isa_ill(s) &&
3373	s->vstart_eq_zero) {
3374	TCGv_ptr src2, mask;
3375	TCGv dst;
3376	TCGv_i32 desc;
3377	uint32_t data = 0;
3378	data = FIELD_DP32(data, VDATA, VM, a->vm);
3379	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3380
3381	mask = tcg_temp_new_ptr();
3382	src2 = tcg_temp_new_ptr();
3383	dst = dest_gpr(s, a->rd);
3384	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
3385	s->cfg_ptr->vlenb, data));
3386
3387	tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, a->rs2));
3388	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
3389
3390	gen_helper_vcpop_m(dst, mask, src2, tcg_env, desc);
3391	gen_set_gpr(s, a->rd, dst);
3392	return true;
3393	}
3394	return false;
3395	}
3396
3397	/* vmfirst find-first-set mask bit */
3398	static bool trans_vfirst_m(DisasContext s, arg_rmr a)
3399	{
3400	if (require_rvv(s) &&
3401	vext_check_isa_ill(s) &&
3402	s->vstart_eq_zero) {
3403	TCGv_ptr src2, mask;
3404	TCGv dst;
3405	TCGv_i32 desc;
3406	uint32_t data = 0;
3407	data = FIELD_DP32(data, VDATA, VM, a->vm);
3408	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3409
3410	mask = tcg_temp_new_ptr();
3411	src2 = tcg_temp_new_ptr();
3412	dst = dest_gpr(s, a->rd);
3413	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
3414	s->cfg_ptr->vlenb, data));
3415
3416	tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, a->rs2));
3417	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0));
3418
3419	gen_helper_vfirst_m(dst, mask, src2, tcg_env, desc);
3420	gen_set_gpr(s, a->rd, dst);
3421	return true;
3422	}
3423	return false;
3424	}
3425
3426	/*
3427	* vmsbf.m set-before-first mask bit
3428	* vmsif.m set-including-first mask bit
3429	* vmsof.m set-only-first mask bit
3430	*/
3431	#define GEN_M_TRANS(NAME) \
3432	static bool trans_##NAME(DisasContext s, arg_rmr a) \
3433	{ \
3434	if (require_rvv(s) && \
3435	vext_check_isa_ill(s) && \
3436	require_vm(a->vm, a->rd) && \
3437	(a->rd != a->rs2) && \
3438	s->vstart_eq_zero) { \
3439	uint32_t data = 0; \
3440	gen_helper_gvec_3_ptr *fn = gen_helper_##NAME; \
3441	\
3442	data = FIELD_DP32(data, VDATA, VM, a->vm); \
3443	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3444	data = \
3445	FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\
3446	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
3447	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), \
3448	vreg_ofs(s, 0), vreg_ofs(s, a->rs2), \
3449	tcg_env, s->cfg_ptr->vlenb, \
3450	s->cfg_ptr->vlenb, \
3451	data, fn); \
3452	finalize_rvv_inst(s); \
3453	return true; \
3454	} \
3455	return false; \
3456	}
3457
3458	GEN_M_TRANS(vmsbf_m)
3459	GEN_M_TRANS(vmsif_m)
3460	GEN_M_TRANS(vmsof_m)
3461
3462	/*
3463	* Vector Iota Instruction
3464	*
3465	* 1. The destination register cannot overlap the source register.
3466	* 2. If masked, cannot overlap the mask register ('v0').
3467	* 3. An illegal instruction exception is raised if vstart is non-zero.
3468	*/
3469	static bool trans_viota_m(DisasContext s, arg_viota_m a)
3470	{
3471	if (require_rvv(s) &&
3472	vext_check_isa_ill(s) &&
3473	!is_overlapped(a->rd, 1 << MAX(s->lmul, 0), a->rs2, 1) &&
3474	require_vm(a->vm, a->rd) &&
3475	require_align(a->rd, s->lmul) &&
3476	s->vstart_eq_zero) {
3477	uint32_t data = 0;
3478
3479	data = FIELD_DP32(data, VDATA, VM, a->vm);
3480	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3481	data = FIELD_DP32(data, VDATA, VTA, s->vta);
3482	data = FIELD_DP32(data, VDATA, VMA, s->vma);
3483	static gen_helper_gvec_3_ptr * const fns[4] = {
3484	gen_helper_viota_m_b, gen_helper_viota_m_h,
3485	gen_helper_viota_m_w, gen_helper_viota_m_d,
3486	};
3487	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
3488	vreg_ofs(s, a->rs2), tcg_env,
3489	s->cfg_ptr->vlenb,
3490	s->cfg_ptr->vlenb, data, fns[s->sew]);
3491	finalize_rvv_inst(s);
3492	return true;
3493	}
3494	return false;
3495	}
3496
3497	/* Vector Element Index Instruction */
3498	static bool trans_vid_v(DisasContext s, arg_vid_v a)
3499	{
3500	if (require_rvv(s) &&
3501	vext_check_isa_ill(s) &&
3502	require_align(a->rd, s->lmul) &&
3503	require_vm(a->vm, a->rd)) {
3504	uint32_t data = 0;
3505
3506	data = FIELD_DP32(data, VDATA, VM, a->vm);
3507	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3508	data = FIELD_DP32(data, VDATA, VTA, s->vta);
3509	data = FIELD_DP32(data, VDATA, VMA, s->vma);
3510	static gen_helper_gvec_2_ptr * const fns[4] = {
3511	gen_helper_vid_v_b, gen_helper_vid_v_h,
3512	gen_helper_vid_v_w, gen_helper_vid_v_d,
3513	};
3514	tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
3515	tcg_env, s->cfg_ptr->vlenb,
3516	s->cfg_ptr->vlenb,
3517	data, fns[s->sew]);
3518	finalize_rvv_inst(s);
3519	return true;
3520	}
3521	return false;
3522	}
3523
3524	/*
3525	*** Vector Permutation Instructions
3526	*/
3527
3528	static void load_element(TCGv_i64 dest, TCGv_ptr base,
3529	int ofs, int sew, bool sign)
3530	{
3531	switch (sew) {
3532	case MO_8:
3533	if (!sign) {
3534	tcg_gen_ld8u_i64(dest, base, ofs);
3535	} else {
3536	tcg_gen_ld8s_i64(dest, base, ofs);
3537	}
3538	break;
3539	case MO_16:
3540	if (!sign) {
3541	tcg_gen_ld16u_i64(dest, base, ofs);
3542	} else {
3543	tcg_gen_ld16s_i64(dest, base, ofs);
3544	}
3545	break;
3546	case MO_32:
3547	if (!sign) {
3548	tcg_gen_ld32u_i64(dest, base, ofs);
3549	} else {
3550	tcg_gen_ld32s_i64(dest, base, ofs);
3551	}
3552	break;
3553	case MO_64:
3554	tcg_gen_ld_i64(dest, base, ofs);
3555	break;
3556	default:
3557	g_assert_not_reached();
3558	}
3559	}
3560
3561	/* offset of the idx element with base register r */
3562	static uint32_t endian_ofs(DisasContext *s, int r, int idx)
3563	{
3564	if (HOST_BIG_ENDIAN) {
3565	return vreg_ofs(s, r) + ((idx ^ (7 >> s->sew)) << s->sew);
3566	} else {
3567	return vreg_ofs(s, r) + (idx << s->sew);
3568	}
3569	}
3570
3571	/* adjust the index according to the endian */
3572	static void endian_adjust(TCGv_i32 ofs, int sew)
3573	{
3574	if (HOST_BIG_ENDIAN) {
3575	tcg_gen_xori_i32(ofs, ofs, 7 >> sew);
3576	}
3577	}
3578
3579	/* Load idx >= VLMAX ? 0 : vreg[idx] */
3580	static void vec_element_loadx(DisasContext *s, TCGv_i64 dest,
3581	int vreg, TCGv idx, int vlmax)
3582	{
3583	TCGv_i32 ofs = tcg_temp_new_i32();
3584	TCGv_ptr base = tcg_temp_new_ptr();
3585	TCGv_i64 t_idx = tcg_temp_new_i64();
3586	TCGv_i64 t_vlmax, t_zero;
3587
3588	/*
3589	* Mask the index to the length so that we do
3590	* not produce an out-of-range load.
3591	*/
3592	tcg_gen_trunc_tl_i32(ofs, idx);
3593	tcg_gen_andi_i32(ofs, ofs, vlmax - 1);
3594
3595	/* Convert the index to an offset. */
3596	endian_adjust(ofs, s->sew);
3597	tcg_gen_shli_i32(ofs, ofs, s->sew);
3598
3599	/* Convert the index to a pointer. */
3600	tcg_gen_ext_i32_ptr(base, ofs);
3601	tcg_gen_add_ptr(base, base, tcg_env);
3602
3603	/* Perform the load. */
3604	load_element(dest, base,
3605	vreg_ofs(s, vreg), s->sew, false);
3606
3607	/* Flush out-of-range indexing to zero. */
3608	t_vlmax = tcg_constant_i64(vlmax);
3609	t_zero = tcg_constant_i64(0);
3610	tcg_gen_extu_tl_i64(t_idx, idx);
3611
3612	tcg_gen_movcond_i64(TCG_COND_LTU, dest, t_idx,
3613	t_vlmax, dest, t_zero);
3614	}
3615
3616	static void vec_element_loadi(DisasContext *s, TCGv_i64 dest,
3617	int vreg, int idx, bool sign)
3618	{
3619	load_element(dest, tcg_env, endian_ofs(s, vreg, idx), s->sew, sign);
3620	}
3621
3622	typedef void gen_helper_vset_velem0(TCGv_ptr, TCGv_i64, TCGv_env, TCGv_i32);
3623
3624	static void vec_element_storei_tail(DisasContext *s, int vreg, TCGv_i64 val)
3625	{
3626	static gen_helper_vset_velem0 * const fns[4] = {
3627	gen_helper_vset_velem0_b, gen_helper_vset_velem0_h,
3628	gen_helper_vset_velem0_w, gen_helper_vset_velem0_d,
3629	};
3630	TCGv_ptr dest = tcg_temp_new_ptr();
3631	uint32_t data = FIELD_DP32(0, VDATA, VTA, s->vta);
3632	TCGv_i32 desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb,
3633	s->cfg_ptr->vlenb, data));
3634
3635	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, vreg));
3636	fns[s->sew](dest, val, tcg_env, desc);
3637	}
3638
3639	/* vmv.x.s rd, vs2 # x[rd] = vs2[0] */
3640	static bool trans_vmv_x_s(DisasContext s, arg_vmv_x_s a)
3641	{
3642	if (require_rvv(s) &&
3643	vext_check_isa_ill(s)) {
3644	TCGv_i64 t1;
3645	TCGv dest;
3646
3647	t1 = tcg_temp_new_i64();
3648	dest = tcg_temp_new();
3649	/*
3650	* load vreg and sign-extend to 64 bits,
3651	* then truncate to XLEN bits before storing to gpr.
3652	*/
3653	vec_element_loadi(s, t1, a->rs2, 0, true);
3654	tcg_gen_trunc_i64_tl(dest, t1);
3655	gen_set_gpr(s, a->rd, dest);
3656	tcg_gen_movi_i32(cpu_vstart, 0);
3657	finalize_rvv_inst(s);
3658	return true;
3659	}
3660	return false;
3661	}
3662
3663	/* vmv.s.x vd, rs1 # vd[0] = rs1 */
3664	static bool trans_vmv_s_x(DisasContext s, arg_vmv_s_x a)
3665	{
3666	if (require_rvv(s) &&
3667	vext_check_isa_ill(s)) {
3668	/* This instruction ignores LMUL and vector register groups */
3669	TCGv_i64 t1;
3670	TCGv s1;
3671	TCGLabel *over = gen_new_label();
3672
3673	tcg_gen_brcond_i32(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
3674
3675	t1 = tcg_temp_new_i64();
3676
3677	/*
3678	* load gpr and sign-extend to 64 bits,
3679	* then truncate to SEW bits when storing to vreg.
3680	*/
3681	s1 = get_gpr(s, a->rs1, EXT_NONE);
3682	tcg_gen_ext_tl_i64(t1, s1);
3683	vec_element_storei_tail(s, a->rd, t1);
3684	gen_set_label(over);
3685	tcg_gen_movi_i32(cpu_vstart, 0);
3686	finalize_rvv_inst(s);
3687	return true;
3688	}
3689	return false;
3690	}
3691
3692	/* Floating-Point Scalar Move Instructions */
3693	static bool trans_vfmv_f_s(DisasContext s, arg_vfmv_f_s a)
3694	{
3695	if (require_rvv(s) &&
3696	require_rvf(s) &&
3697	vext_check_isa_ill(s)) {
3698	gen_set_rm(s, RISCV_FRM_DYN);
3699
3700	unsigned int ofs = (8 << s->sew);
3701	unsigned int len = 64 - ofs;
3702	TCGv_i64 t_nan;
3703
3704	vec_element_loadi(s, cpu_fpr[a->rd], a->rs2, 0, false);
3705	/* NaN-box f[rd] as necessary for SEW */
3706	if (len) {
3707	t_nan = tcg_constant_i64(UINT64_MAX);
3708	tcg_gen_deposit_i64(cpu_fpr[a->rd], cpu_fpr[a->rd],
3709	t_nan, ofs, len);
3710	}
3711
3712	mark_fs_dirty(s);
3713	tcg_gen_movi_i32(cpu_vstart, 0);
3714	finalize_rvv_inst(s);
3715	return true;
3716	}
3717	return false;
3718	}
3719
3720	/* vfmv.s.f vd, rs1 # vd[0] = rs1 (vs2=0) */
3721	static bool trans_vfmv_s_f(DisasContext s, arg_vfmv_s_f a)
3722	{
3723	if (require_rvv(s) &&
3724	require_rvf(s) &&
3725	vext_check_isa_ill(s) &&
3726	vext_check_altfmt(s, MO_16)) {
3727	gen_set_rm(s, RISCV_FRM_DYN);
3728
3729	/* The instructions ignore LMUL and vector register group. */
3730	TCGv_i64 t1;
3731	TCGLabel *over = gen_new_label();
3732
3733	/* if vstart >= vl, skip vector register write back */
3734	tcg_gen_brcond_i32(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
3735
3736	/* NaN-box f[rs1] */
3737	t1 = tcg_temp_new_i64();
3738	do_nanbox(s, t1, cpu_fpr[a->rs1]);
3739
3740	vec_element_storei_tail(s, a->rd, t1);
3741
3742	gen_set_label(over);
3743	tcg_gen_movi_i32(cpu_vstart, 0);
3744	finalize_rvv_inst(s);
3745	return true;
3746	}
3747	return false;
3748	}
3749
3750	/* Vector Slide Instructions */
3751	static bool slideup_check(DisasContext s, arg_rmrr a)
3752	{
3753	return require_rvv(s) &&
3754	vext_check_isa_ill(s) &&
3755	vext_check_slide(s, a->rd, a->rs2, a->vm, true);
3756	}
3757
3758	GEN_OPIVX_TRANS(vslideup_vx, slideup_check)
3759	GEN_OPIVI_TRANS(vslideup_vi, IMM_ZX, vslideup_vx, slideup_check)
3760
3761	static bool slidedown_check(DisasContext s, arg_rmrr a)
3762	{
3763	return require_rvv(s) &&
3764	vext_check_isa_ill(s) &&
3765	vext_check_slide(s, a->rd, a->rs2, a->vm, false);
3766	}
3767
3768	GEN_OPIVX_TRANS(vslidedown_vx, slidedown_check)
3769	GEN_OPIVI_TRANS(vslidedown_vi, IMM_ZX, vslidedown_vx, slidedown_check)
3770
3771	typedef void gen_helper_vslide1_vx(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_ptr,
3772	TCGv_env, TCGv_i32);
3773
3774	#define GEN_OPIVX_VSLIDE1_TRANS(NAME, CHECK) \
3775	static bool trans_##NAME(DisasContext s, arg_rmrr a) \
3776	{ \
3777	if (CHECK(s, a)) { \
3778	static gen_helper_vslide1_vx * const fns[4] = { \
3779	gen_helper_##NAME##_b, gen_helper_##NAME##_h, \
3780	gen_helper_##NAME##_w, gen_helper_##NAME##_d, \
3781	}; \
3782	\
3783	TCGv_ptr dest, src2, mask; \
3784	TCGv_i64 src1; \
3785	TCGv_i32 desc; \
3786	uint32_t data = 0; \
3787	\
3788	dest = tcg_temp_new_ptr(); \
3789	mask = tcg_temp_new_ptr(); \
3790	src2 = tcg_temp_new_ptr(); \
3791	src1 = tcg_temp_new_i64(); \
3792	\
3793	data = FIELD_DP32(data, VDATA, VM, a->vm); \
3794	data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \
3795	data = FIELD_DP32(data, VDATA, VTA, s->vta); \
3796	data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); \
3797	data = FIELD_DP32(data, VDATA, VMA, s->vma); \
3798	desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, \
3799	s->cfg_ptr->vlenb, data)); \
3800	\
3801	tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd)); \
3802	tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, a->rs2)); \
3803	tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0)); \
3804	tcg_gen_ext_tl_i64(src1, get_gpr(s, a->rs1, EXT_SIGN)); \
3805	\
3806	fns[s->sew](dest, mask, src1, src2, tcg_env, desc); \
3807	\
3808	tcg_gen_movi_i32(cpu_vstart, 0); \
3809	finalize_rvv_inst(s); \
3810	\
3811	return true; \
3812	} \
3813	return false; \
3814	}
3815
3816	GEN_OPIVX_VSLIDE1_TRANS(vslide1up_vx, slideup_check)
3817	GEN_OPIVX_VSLIDE1_TRANS(vslide1down_vx, slidedown_check)
3818
3819	/* Vector Floating-Point Slide Instructions */
3820	static bool fslideup_check(DisasContext s, arg_rmrr a,
3821	int8_t valid_bfa_vsew)
3822	{
3823	return slideup_check(s, a) &&
3824	require_rvf(s) &&
3825	vext_check_altfmt(s, valid_bfa_vsew);
3826	}
3827
3828	static bool fslidedown_check(DisasContext s, arg_rmrr a,
3829	int8_t valid_bfa_vsew)
3830	{
3831	return slidedown_check(s, a) &&
3832	require_rvf(s) &&
3833	vext_check_altfmt(s, valid_bfa_vsew);
3834	}
3835
3836	GEN_OPFVF_BFA_TRANS(vfslide1up_vf, fslideup_check, vfslide1up_vf_h)
3837	GEN_OPFVF_BFA_TRANS(vfslide1down_vf, fslidedown_check, vfslide1down_vf_h)
3838
3839	/* Vector Register Gather Instruction */
3840	static bool vrgather_vv_check(DisasContext s, arg_rmrr a)
3841	{
3842	return require_rvv(s) &&
3843	vext_check_isa_ill(s) &&
3844	vext_check_input_eew(s, a->rs1, s->sew, a->rs2, s->sew, a->vm) &&
3845	require_align(a->rd, s->lmul) &&
3846	require_align(a->rs1, s->lmul) &&
3847	require_align(a->rs2, s->lmul) &&
3848	(a->rd != a->rs2 && a->rd != a->rs1) &&
3849	require_vm(a->vm, a->rd);
3850	}
3851
3852	static bool vrgatherei16_vv_check(DisasContext s, arg_rmrr a)
3853	{
3854	int8_t emul = MO_16 - s->sew + s->lmul;
3855	return require_rvv(s) &&
3856	vext_check_isa_ill(s) &&
3857	vext_check_input_eew(s, a->rs1, MO_16, a->rs2, s->sew, a->vm) &&
3858	(emul >= -3 && emul <= 3) &&
3859	require_align(a->rd, s->lmul) &&
3860	require_align(a->rs1, emul) &&
3861	require_align(a->rs2, s->lmul) &&
3862	(a->rd != a->rs2 && a->rd != a->rs1) &&
3863	!is_overlapped(a->rd, 1 << MAX(s->lmul, 0),
3864	a->rs1, 1 << MAX(emul, 0)) &&
3865	!is_overlapped(a->rd, 1 << MAX(s->lmul, 0),
3866	a->rs2, 1 << MAX(s->lmul, 0)) &&
3867	require_vm(a->vm, a->rd);
3868	}
3869
3870	GEN_OPIVV_TRANS(vrgather_vv, vrgather_vv_check)
3871	GEN_OPIVV_TRANS(vrgatherei16_vv, vrgatherei16_vv_check)
3872
3873	static bool vrgather_vx_check(DisasContext s, arg_rmrr a)
3874	{
3875	return require_rvv(s) &&
3876	vext_check_isa_ill(s) &&
3877	vext_check_input_eew(s, -1, MO_64, a->rs2, s->sew, a->vm) &&
3878	require_align(a->rd, s->lmul) &&
3879	require_align(a->rs2, s->lmul) &&
3880	(a->rd != a->rs2) &&
3881	require_vm(a->vm, a->rd);
3882	}
3883
3884	/* vrgather.vx vd, vs2, rs1, vm # vd[i] = (x[rs1] >= VLMAX) ? 0 : vs2[rs1] */
3885	static bool trans_vrgather_vx(DisasContext s, arg_rmrr a)
3886	{
3887	if (!vrgather_vx_check(s, a)) {
3888	return false;
3889	}
3890
3891	if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
3892	int vlmax = vext_get_vlmax(s->cfg_ptr->vlenb, s->sew, s->lmul);
3893	TCGv_i64 dest = tcg_temp_new_i64();
3894
3895	if (a->rs1 == 0) {
3896	vec_element_loadi(s, dest, a->rs2, 0, false);
3897	} else {
3898	vec_element_loadx(s, dest, a->rs2, cpu_gpr[a->rs1], vlmax);
3899	}
3900
3901	tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd),
3902	MAXSZ(s), MAXSZ(s), dest);
3903	finalize_rvv_inst(s);
3904	} else {
3905	static gen_helper_opivx * const fns[4] = {
3906	gen_helper_vrgather_vx_b, gen_helper_vrgather_vx_h,
3907	gen_helper_vrgather_vx_w, gen_helper_vrgather_vx_d
3908	};
3909	return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew], s);
3910	}
3911	return true;
3912	}
3913
3914	/* vrgather.vi vd, vs2, imm, vm # vd[i] = (imm >= VLMAX) ? 0 : vs2[imm] */
3915	static bool trans_vrgather_vi(DisasContext s, arg_rmrr a)
3916	{
3917	if (!vrgather_vx_check(s, a)) {
3918	return false;
3919	}
3920
3921	if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
3922	int vlmax = vext_get_vlmax(s->cfg_ptr->vlenb, s->sew, s->lmul);
3923	if (a->rs1 >= vlmax) {
3924	tcg_gen_gvec_dup_imm(MO_64, vreg_ofs(s, a->rd),
3925	MAXSZ(s), MAXSZ(s), 0);
3926	} else {
3927	tcg_gen_gvec_dup_mem(s->sew, vreg_ofs(s, a->rd),
3928	endian_ofs(s, a->rs2, a->rs1),
3929	MAXSZ(s), MAXSZ(s));
3930	}
3931	finalize_rvv_inst(s);
3932	} else {
3933	static gen_helper_opivx * const fns[4] = {
3934	gen_helper_vrgather_vx_b, gen_helper_vrgather_vx_h,
3935	gen_helper_vrgather_vx_w, gen_helper_vrgather_vx_d
3936	};
3937	return opivi_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew],
3938	s, IMM_ZX);
3939	}
3940	return true;
3941	}
3942
3943	/*
3944	* Vector Compress Instruction
3945	*
3946	* The destination vector register group cannot overlap the
3947	* source vector register group or the source mask register.
3948	*/
3949	static bool vcompress_vm_check(DisasContext s, arg_r a)
3950	{
3951	return require_rvv(s) &&
3952	vext_check_isa_ill(s) &&
3953	require_align(a->rd, s->lmul) &&
3954	require_align(a->rs2, s->lmul) &&
3955	(a->rd != a->rs2) &&
3956	!is_overlapped(a->rd, 1 << MAX(s->lmul, 0), a->rs1, 1) &&
3957	s->vstart_eq_zero;
3958	}
3959
3960	static bool trans_vcompress_vm(DisasContext s, arg_r a)
3961	{
3962	if (vcompress_vm_check(s, a)) {
3963	uint32_t data = 0;
3964	static gen_helper_gvec_4_ptr * const fns[4] = {
3965	gen_helper_vcompress_vm_b, gen_helper_vcompress_vm_h,
3966	gen_helper_vcompress_vm_w, gen_helper_vcompress_vm_d,
3967	};
3968
3969	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3970	data = FIELD_DP32(data, VDATA, VTA, s->vta);
3971	tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
3972	vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2),
3973	tcg_env, s->cfg_ptr->vlenb,
3974	s->cfg_ptr->vlenb, data,
3975	fns[s->sew]);
3976	finalize_rvv_inst(s);
3977	return true;
3978	}
3979	return false;
3980	}
3981
3982	/*
3983	* Whole Vector Register Move Instructions depend on vtype register(vsew).
3984	* Thus, we need to check vill bit. (Section 16.6)
3985	*/
3986	#define GEN_VMV_WHOLE_TRANS(NAME, LEN) \
3987	static bool trans_##NAME(DisasContext s, arg_##NAME a) \
3988	{ \
3989	if (require_rvv(s) && \
3990	vext_check_isa_ill(s) && \
3991	QEMU_IS_ALIGNED(a->rd, LEN) && \
3992	QEMU_IS_ALIGNED(a->rs2, LEN)) { \
3993	uint32_t maxsz = s->cfg_ptr->vlenb * LEN; \
3994	if (s->vstart_eq_zero) { \
3995	tcg_gen_gvec_mov(s->sew, vreg_ofs(s, a->rd), \
3996	vreg_ofs(s, a->rs2), maxsz, maxsz); \
3997	} else { \
3998	tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), \
3999	tcg_env, maxsz, maxsz, 0, gen_helper_vmvr_v); \
4000	} \
4001	finalize_rvv_inst(s); \
4002	return true; \
4003	} \
4004	return false; \
4005	}
4006
4007	GEN_VMV_WHOLE_TRANS(vmv1r_v, 1)
4008	GEN_VMV_WHOLE_TRANS(vmv2r_v, 2)
4009	GEN_VMV_WHOLE_TRANS(vmv4r_v, 4)
4010	GEN_VMV_WHOLE_TRANS(vmv8r_v, 8)
4011
4012	static bool int_ext_check(DisasContext s, arg_rmr a, uint8_t div)
4013	{
4014	uint8_t from = (s->sew + 3) - div;
4015	bool ret = require_rvv(s) &&
4016	(from >= 3 && from <= 8) &&
4017	(a->rd != a->rs2) &&
4018	require_align(a->rd, s->lmul) &&
4019	require_align(a->rs2, s->lmul - div) &&
4020	require_vm(a->vm, a->rd) &&
4021	require_noover(a->rd, s->lmul, a->rs2, s->lmul - div) &&
4022	vext_check_input_eew(s, -1, 0, a->rs2, s->sew, a->vm);
4023
4024	return ret;
4025	}
4026
4027	static bool int_ext_op(DisasContext s, arg_rmr a, uint8_t seq)
4028	{
4029	uint32_t data = 0;
4030	gen_helper_gvec_3_ptr *fn;
4031
4032	static gen_helper_gvec_3_ptr * const fns[6][4] = {
4033	{
4034	NULL, gen_helper_vzext_vf2_h,
4035	gen_helper_vzext_vf2_w, gen_helper_vzext_vf2_d
4036	},
4037	{
4038	NULL, NULL,
4039	gen_helper_vzext_vf4_w, gen_helper_vzext_vf4_d,
4040	},
4041	{
4042	NULL, NULL,
4043	NULL, gen_helper_vzext_vf8_d
4044	},
4045	{
4046	NULL, gen_helper_vsext_vf2_h,
4047	gen_helper_vsext_vf2_w, gen_helper_vsext_vf2_d
4048	},
4049	{
4050	NULL, NULL,
4051	gen_helper_vsext_vf4_w, gen_helper_vsext_vf4_d,
4052	},
4053	{
4054	NULL, NULL,
4055	NULL, gen_helper_vsext_vf8_d
4056	}
4057	};
4058
4059	fn = fns[seq][s->sew];
4060	if (fn == NULL) {
4061	return false;
4062	}
4063
4064	data = FIELD_DP32(data, VDATA, VM, a->vm);
4065	data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
4066	data = FIELD_DP32(data, VDATA, VTA, s->vta);
4067	data = FIELD_DP32(data, VDATA, VMA, s->vma);
4068
4069	tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
4070	vreg_ofs(s, a->rs2), tcg_env,
4071	s->cfg_ptr->vlenb,
4072	s->cfg_ptr->vlenb, data, fn);
4073
4074	finalize_rvv_inst(s);
4075	return true;
4076	}
4077
4078	/* Vector Integer Extension */
4079	#define GEN_INT_EXT_TRANS(NAME, DIV, SEQ) \
4080	static bool trans_##NAME(DisasContext s, arg_rmr a) \
4081	{ \
4082	if (int_ext_check(s, a, DIV)) { \
4083	return int_ext_op(s, a, SEQ); \
4084	} \
4085	return false; \
4086	}
4087
4088	GEN_INT_EXT_TRANS(vzext_vf2, 1, 0)
4089	GEN_INT_EXT_TRANS(vzext_vf4, 2, 1)
4090	GEN_INT_EXT_TRANS(vzext_vf8, 3, 2)
4091	GEN_INT_EXT_TRANS(vsext_vf2, 1, 3)
4092	GEN_INT_EXT_TRANS(vsext_vf4, 2, 4)
4093	GEN_INT_EXT_TRANS(vsext_vf8, 3, 5)