vector_helper.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 5,908 lines 232 KB

Raw

1	/*
2	* RISC-V Vector Extension Helpers for QEMU.
3	*
4	* Copyright (c) 2020 T-Head Semiconductor Co., Ltd. All rights reserved.
5	*
6	* This program is free software; you can redistribute it and/or modify it
7	* under the terms and conditions of the GNU General Public License,
8	* version 2 or later, as published by the Free Software Foundation.
9	*
10	* This program is distributed in the hope it will be useful, but WITHOUT
11	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13	* more details.
14	*
15	* You should have received a copy of the GNU General Public License along with
16	* this program. If not, see <http://www.gnu.org/licenses/>.
17	*/
18
19	#include "qemu/osdep.h"
20	#include "qemu/host-utils.h"
21	#include "qemu/bitops.h"
22	#include "cpu.h"
23	#include "exec/memop.h"
24	#include "accel/tcg/cpu-ldst.h"
25	#include "accel/tcg/probe.h"
26	#include "exec/page-protection.h"
27	#include "exec/helper-proto.h"
28	#include "exec/tlb-flags.h"
29	#include "exec/target_page.h"
30	#include "fpu/softfloat.h"
31	#include "tcg/tcg-gvec-desc.h"
32	#include "internals.h"
33	#include "vector_internals.h"
34	#include <math.h>
35
36	static target_ulong vtype_reserved(CPURISCVState *env, target_ulong vtype)
37	{
38	int xlen = riscv_cpu_xlen(env);
39	target_ulong reserved = 0;
40
41	if (riscv_cpu_cfg(env)->ext_zvfbfa) {
42	reserved = vtype & MAKE_64BIT_MASK(R_VTYPE_RESERVED_SHIFT,
43	xlen - 1 - R_VTYPE_RESERVED_SHIFT);
44	} else {
45	reserved = vtype & MAKE_64BIT_MASK(R_VTYPE_ALTFMT_SHIFT,
46	xlen - 1 - R_VTYPE_ALTFMT_SHIFT);
47	}
48
49	return reserved;
50	}
51
52	static inline void reset_ill_vtype(CPURISCVState *env)
53	{
54	/* only set vill bit. */
55	env->vill = 1;
56	env->vtype = 0;
57	env->vl = 0;
58	env->vstart = 0;
59	}
60
61	target_ulong HELPER(vsetvl)(CPURISCVState *env, target_ulong s1,
62	target_ulong s2, target_ulong x0)
63	{
64	int vlmax, vl;
65	RISCVCPU *cpu = env_archcpu(env);
66	uint64_t vlmul = FIELD_EX64(s2, VTYPE, VLMUL);
67	uint8_t vsew = FIELD_EX64(s2, VTYPE, VSEW);
68	uint16_t sew = 8 << vsew;
69	uint8_t altfmt = FIELD_EX64(s2, VTYPE, ALTFMT);
70	bool ill_altfmt = true;
71	int xlen = riscv_cpu_xlen(env);
72	bool vill = (s2 >> (xlen - 1)) & 0x1;
73	int8_t lmul;
74
75	if (vlmul & 4) {
76	/*
77	* Fractional LMUL, check:
78	*
79	* ELEN * LMUL >= SEW
80	* ELEN >> (8 - vlmul) >= sew
81	*/
82	if (vlmul == 4 \|\|
83	(cpu->cfg.elen >> (8 - vlmul)) < sew) {
84	vill = true;
85	}
86	}
87
88	switch (vsew) {
89	case MO_8:
90	ill_altfmt &= !(cpu->cfg.ext_zvfbfa);
91	break;
92	case MO_16:
93	ill_altfmt &= !(cpu->cfg.ext_zvfbfa);
94	break;
95	default:
96	break;
97	}
98
99	if (altfmt && ill_altfmt) {
100	vill = true;
101	}
102
103	if ((sew > cpu->cfg.elen) \|\| vill \|\| (vtype_reserved(env, s2) != 0)) {
104	reset_ill_vtype(env);
105	return 0;
106	}
107
108	/* lmul encoded as in DisasContext::lmul */
109	lmul = sextract32(FIELD_EX64(s2, VTYPE, VLMUL), 0, 3);
110	vlmax = vext_get_vlmax(cpu->cfg.vlenb, vsew, lmul);
111	if (s1 <= vlmax) {
112	vl = s1;
113	} else if (s1 < 2 * vlmax && cpu->cfg.rvv_vl_half_avl) {
114	vl = (s1 + 1) >> 1;
115	} else {
116	vl = vlmax;
117	}
118
119	if (cpu->cfg.rvv_vsetvl_x0_vill && x0 && (env->vl != vl)) {
120	reset_ill_vtype(env);
121	return 0;
122	}
123
124	env->vl = vl;
125	env->vtype = s2;
126	env->vstart = 0;
127	env->vill = 0;
128	return vl;
129	}
130
131	/*
132	* Get the maximum number of elements can be operated.
133	*
134	* log2_esz: log2 of element size in bytes.
135	*/
136	static inline uint32_t vext_max_elems(uint32_t desc, uint32_t log2_esz)
137	{
138	/*
139	* As simd_desc support at most 2048 bytes, the max vlen is 1024 bits.
140	* so vlen in bytes (vlenb) is encoded as maxsz.
141	*/
142	uint32_t vlenb = simd_maxsz(desc);
143
144	/* Return VLMAX */
145	int scale = vext_lmul(desc) - log2_esz;
146	return scale < 0 ? vlenb >> -scale : vlenb << scale;
147	}
148
149	/*
150	* This function checks watchpoint before real load operation.
151	*
152	* In system mode, the TLB API probe_access is enough for watchpoint check.
153	* In user mode, there is no watchpoint support now.
154	*
155	* It will trigger an exception if there is no mapping in TLB
156	* and page table walk can't fill the TLB entry. Then the guest
157	* software can return here after process the exception or never return.
158	*
159	* This function can also be used when direct access to probe_access_flags is
160	* needed in order to access the flags. If a pointer to a flags operand is
161	* provided the function will call probe_access_flags instead, use nonfault
162	* and update host and flags.
163	*/
164	static void probe_pages(CPURISCVState *env, target_ulong addr, target_ulong len,
165	uintptr_t ra, MMUAccessType access_type, int mmu_index,
166	void *host, int flags, bool nonfault)
167	{
168	target_ulong pagelen = -(addr \| TARGET_PAGE_MASK);
169	target_ulong curlen = MIN(pagelen, len);
170
171	if (flags != NULL) {
172	*flags = probe_access_flags(env, adjust_addr(env, addr), curlen,
173	access_type, mmu_index, nonfault, host, ra);
174	} else {
175	probe_access(env, adjust_addr(env, addr), curlen, access_type,
176	mmu_index, ra);
177	}
178
179	if (len > curlen) {
180	addr += curlen;
181	curlen = len - curlen;
182	if (flags != NULL) {
183	*flags \|= probe_access_flags(env, adjust_addr(env, addr), curlen,
184	access_type, mmu_index, nonfault,
185	host, ra);
186	} else {
187	probe_access(env, adjust_addr(env, addr), curlen, access_type,
188	mmu_index, ra);
189	}
190	}
191	}
192
193	static inline void vext_set_elem_mask(void *v0, int index,
194	uint8_t value)
195	{
196	int idx = index / 64;
197	int pos = index % 64;
198	uint64_t old = ((uint64_t *)v0)[idx];
199	((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value);
200	}
201
202	static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size)
203	{
204	int mmu_idx = riscv_env_mmu_index(env, false);
205	MemOp memop = size_memop(size) \| mo_endian_env(env);
206
207	if (!riscv_cpu_cfg(env)->ext_zicclsm) {
208	memop \|= MO_ALIGN;
209	}
210
211	return make_memop_idx(memop, mmu_idx);
212	}
213
214	/* elements operations for load and store */
215	typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr,
216	uint32_t idx, void *vd, uintptr_t retaddr);
217	typedef void vext_ldst_elem_fn_host(void vd, uint32_t idx, void host);
218
219	#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
220	static inline QEMU_ALWAYS_INLINE \
221	void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
222	uint32_t idx, void *vd, uintptr_t retaddr) \
223	{ \
224	ETYPE cur = ((ETYPE )vd + H(idx)); \
225	MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \
226	*cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
227	} \
228
229	#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
230	static inline QEMU_ALWAYS_INLINE \
231	void NAME##_host(void vd, uint32_t idx, void host) \
232	{ \
233	ETYPE cur = ((ETYPE )vd + H(idx)); \
234	*cur = (ETYPE)LDSUF##_p(host); \
235	}
236
237	GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t, H1, ldb)
238	GEN_VEXT_TLB_LD_ELEM(lde_h, uint16_t, H2, ldw)
239	GEN_VEXT_TLB_LD_ELEM(lde_w, uint32_t, H4, ldl)
240	GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq)
241
242	GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t, H1, ldub)
243	GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le)
244	GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
245	GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
246
247	#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \
248	static inline QEMU_ALWAYS_INLINE \
249	void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
250	uint32_t idx, void *vd, uintptr_t retaddr) \
251	{ \
252	ETYPE data = ((ETYPE )vd + H(idx)); \
253	MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \
254	cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \
255	} \
256
257	#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
258	static inline QEMU_ALWAYS_INLINE \
259	void NAME##_host(void vd, uint32_t idx, void host) \
260	{ \
261	ETYPE data = ((ETYPE )vd + H(idx)); \
262	STSUF##_p(host, data); \
263	}
264
265	GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t, H1, stb)
266	GEN_VEXT_TLB_ST_ELEM(ste_h, uint16_t, H2, stw)
267	GEN_VEXT_TLB_ST_ELEM(ste_w, uint32_t, H4, stl)
268	GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq)
269
270	GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t, H1, stb)
271	GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le)
272	GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le)
273	GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le)
274
275	static inline QEMU_ALWAYS_INLINE void
276	vext_continuous_ldst_tlb(CPURISCVState env, vext_ldst_elem_fn_tlb ldst_tlb,
277	void *vd, uint32_t evl, target_ulong addr,
278	uint32_t reg_start, uintptr_t ra, uint32_t esz,
279	bool is_load)
280	{
281	uint32_t i;
282	for (i = env->vstart; i < evl; env->vstart = ++i, addr += esz) {
283	ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
284	}
285	}
286
287	static inline QEMU_ALWAYS_INLINE void
288	vext_continuous_ldst_host(CPURISCVState env, vext_ldst_elem_fn_host ldst_host,
289	void vd, uint32_t evl, uint32_t reg_start, void host,
290	uint32_t esz, bool is_load)
291	{
292	if (HOST_BIG_ENDIAN) {
293	for (; reg_start < evl; reg_start++, host += esz) {
294	ldst_host(vd, reg_start, host);
295	}
296	} else {
297	if (esz == 1) {
298	uint32_t byte_offset = reg_start * esz;
299	uint32_t size = (evl - reg_start) * esz;
300
301	if (is_load) {
302	memcpy(vd + byte_offset, host, size);
303	} else {
304	memcpy(host, vd + byte_offset, size);
305	}
306	} else {
307	for (; reg_start < evl; reg_start++, host += esz) {
308	ldst_host(vd, reg_start, host);
309	}
310	}
311	}
312	}
313
314	static void vext_set_tail_elems_1s(uint32_t vl, void *vd,
315	uint32_t desc, uint32_t nf,
316	uint32_t esz, uint32_t max_elems)
317	{
318	uint32_t vta = vext_vta(desc);
319	int k;
320
321	if (vta == 0) {
322	return;
323	}
324
325	for (k = 0; k < nf; ++k) {
326	vext_set_elems_1s(vd, vta, (k * max_elems + vl) * esz,
327	(k * max_elems + max_elems) * esz);
328	}
329	}
330
331	/*
332	* stride: access vector element from strided memory
333	*/
334	static void
335	vext_ldst_stride(void vd, void v0, target_ulong base, target_ulong stride,
336	CPURISCVState *env, uint32_t desc, uint32_t vm,
337	vext_ldst_elem_fn_tlb *ldst_elem, uint32_t log2_esz,
338	uintptr_t ra)
339	{
340	uint32_t i, k;
341	uint32_t nf = vext_nf(desc);
342	uint32_t max_elems = vext_max_elems(desc, log2_esz);
343	uint32_t esz = 1 << log2_esz;
344	uint32_t vma = vext_vma(desc);
345
346	VSTART_CHECK_EARLY_EXIT(env, env->vl);
347
348	for (i = env->vstart; i < env->vl; env->vstart = ++i) {
349	k = 0;
350	while (k < nf) {
351	if (!vm && !vext_elem_mask(v0, i)) {
352	/* set masked-off elements to 1s */
353	vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
354	(i + k * max_elems + 1) * esz);
355	k++;
356	continue;
357	}
358	target_ulong addr = base + stride * i + (k << log2_esz);
359	ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
360	k++;
361	}
362	}
363	env->vstart = 0;
364
365	vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
366	}
367
368	#define GEN_VEXT_LD_STRIDE(NAME, ETYPE, LOAD_FN) \
369	void HELPER(NAME)(void vd, void v0, target_ulong base, \
370	target_ulong stride, CPURISCVState *env, \
371	uint32_t desc) \
372	{ \
373	uint32_t vm = vext_vm(desc); \
374	vext_ldst_stride(vd, v0, base, stride, env, desc, vm, LOAD_FN, \
375	ctzl(sizeof(ETYPE)), GETPC()); \
376	}
377
378	GEN_VEXT_LD_STRIDE(vlse8_v, int8_t, lde_b_tlb)
379	GEN_VEXT_LD_STRIDE(vlse16_v, int16_t, lde_h_tlb)
380	GEN_VEXT_LD_STRIDE(vlse32_v, int32_t, lde_w_tlb)
381	GEN_VEXT_LD_STRIDE(vlse64_v, int64_t, lde_d_tlb)
382
383	#define GEN_VEXT_ST_STRIDE(NAME, ETYPE, STORE_FN) \
384	void HELPER(NAME)(void vd, void v0, target_ulong base, \
385	target_ulong stride, CPURISCVState *env, \
386	uint32_t desc) \
387	{ \
388	uint32_t vm = vext_vm(desc); \
389	vext_ldst_stride(vd, v0, base, stride, env, desc, vm, STORE_FN, \
390	ctzl(sizeof(ETYPE)), GETPC()); \
391	}
392
393	GEN_VEXT_ST_STRIDE(vsse8_v, int8_t, ste_b_tlb)
394	GEN_VEXT_ST_STRIDE(vsse16_v, int16_t, ste_h_tlb)
395	GEN_VEXT_ST_STRIDE(vsse32_v, int32_t, ste_w_tlb)
396	GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d_tlb)
397
398	/*
399	* unit-stride: access elements stored contiguously in memory
400	*/
401
402	/* unmasked unit-stride load and store operation */
403	static inline QEMU_ALWAYS_INLINE void
404	vext_page_ldst_us(CPURISCVState env, void vd, target_ulong addr,
405	uint32_t elems, uint32_t nf, uint32_t max_elems,
406	uint32_t log2_esz, bool is_load, int mmu_index,
407	vext_ldst_elem_fn_tlb *ldst_tlb,
408	vext_ldst_elem_fn_host *ldst_host, uintptr_t ra)
409	{
410	void *host;
411	int i, k, flags;
412	uint32_t esz = 1 << log2_esz;
413	uint32_t size = (elems * nf) << log2_esz;
414	uint32_t evl = env->vstart + elems;
415	MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
416
417	/*
418	* Maximum vector length is VLMAX == 2^16 == LMUL * VL / SEW, and
419	* occurs for LMUL == 8, SEW == 8, VL == 2^16.
420	*/
421	g_assert(env->vstart < UINT16_MAX && UINT16_MAX - env->vstart >= elems);
422
423	/* Check page permission/pmp/watchpoint/etc. */
424	probe_pages(env, addr, size, ra, access_type, mmu_index, &host, &flags,
425	true);
426
427	bool misaligned = addr & (esz - 1);
428
429	/*
430	* Allow the host fast-pash when:
431	* 1. Page permission/pmp/watchpoint are checked and we have a contigous
432	* host mapping.
433	* 2. Zicclsm is enabled or load/store is not a misaligned access.
434	* Otherwise, we will fall back to the slow TLB-path.
435	*/
436	if (flags == 0 && (riscv_cpu_cfg(env)->ext_zicclsm \|\| !misaligned)) {
437	if (nf == 1) {
438	vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart,
439	host, esz, is_load);
440	} else {
441	for (i = env->vstart; i < evl; ++i) {
442	k = 0;
443	while (k < nf) {
444	ldst_host(vd, i + k * max_elems, host);
445	host += esz;
446	k++;
447	}
448	}
449	}
450	env->vstart += elems;
451	} else {
452	if (nf == 1) {
453	vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr, env->vstart,
454	ra, esz, is_load);
455	} else {
456	/* load bytes from guest memory */
457	for (i = env->vstart; i < evl; env->vstart = ++i) {
458	k = 0;
459	while (k < nf) {
460	ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems,
461	vd, ra);
462	addr += esz;
463	k++;
464	}
465	}
466	}
467	}
468	}
469
470	static inline QEMU_ALWAYS_INLINE void
471	vext_ldst_us(void vd, target_ulong base, CPURISCVState env, uint32_t desc,
472	vext_ldst_elem_fn_tlb *ldst_tlb,
473	vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
474	uint32_t evl, uintptr_t ra, bool is_load)
475	{
476	uint32_t k;
477	target_ulong page_split, elems, addr;
478	uint32_t nf = vext_nf(desc);
479	uint32_t max_elems = vext_max_elems(desc, log2_esz);
480	uint32_t esz = 1 << log2_esz;
481	uint32_t msize = nf * esz;
482	int mmu_index = riscv_env_mmu_index(env, false);
483
484	VSTART_CHECK_EARLY_EXIT(env, evl);
485
486	#if defined(CONFIG_USER_ONLY)
487	/*
488	* For data sizes <= 6 bytes we get better performance by simply calling
489	* vext_continuous_ldst_tlb
490	*/
491	if (nf == 1 && (evl << log2_esz) <= 6) {
492	addr = base + (env->vstart << log2_esz);
493	vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr, env->vstart, ra,
494	esz, is_load);
495
496	env->vstart = 0;
497	vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
498	return;
499	}
500	#endif
501
502	/* Calculate the page range of first page */
503	addr = base + ((env->vstart * nf) << log2_esz);
504	page_split = -(addr \| TARGET_PAGE_MASK);
505	/* Get number of elements */
506	elems = page_split / msize;
507	if (unlikely(env->vstart + elems >= evl)) {
508	elems = evl - env->vstart;
509	}
510
511	/* Load/store elements in the first page */
512	if (likely(elems)) {
513	vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, log2_esz,
514	is_load, mmu_index, ldst_tlb, ldst_host, ra);
515	}
516
517	/* Load/store elements in the second page */
518	if (unlikely(env->vstart < evl)) {
519	/* Cross page element */
520	if (unlikely(page_split % msize)) {
521	for (k = 0; k < nf; k++) {
522	addr = base + ((env->vstart * nf + k) << log2_esz);
523	ldst_tlb(env, adjust_addr(env, addr),
524	env->vstart + k * max_elems, vd, ra);
525	}
526	env->vstart++;
527	}
528
529	addr = base + ((env->vstart * nf) << log2_esz);
530	/* Get number of elements of second page */
531	elems = evl - env->vstart;
532
533	/* Load/store elements in the second page */
534	vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, log2_esz,
535	is_load, mmu_index, ldst_tlb, ldst_host, ra);
536	}
537
538	env->vstart = 0;
539	vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
540	}
541
542	/*
543	* masked unit-stride load and store operation will be a special case of
544	* stride, stride = NF * sizeof (ETYPE)
545	*/
546
547	#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
548	void HELPER(NAME##_mask)(void vd, void v0, target_ulong base, \
549	CPURISCVState *env, uint32_t desc) \
550	{ \
551	uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \
552	vext_ldst_stride(vd, v0, base, stride, env, desc, false, \
553	LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \
554	} \
555	\
556	void HELPER(NAME)(void vd, void v0, target_ulong base, \
557	CPURISCVState *env, uint32_t desc) \
558	{ \
559	vext_ldst_us(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
560	ctzl(sizeof(ETYPE)), env->vl, GETPC(), true); \
561	}
562
563	GEN_VEXT_LD_US(vle8_v, int8_t, lde_b_tlb, lde_b_host)
564	GEN_VEXT_LD_US(vle16_v, int16_t, lde_h_tlb, lde_h_host)
565	GEN_VEXT_LD_US(vle32_v, int32_t, lde_w_tlb, lde_w_host)
566	GEN_VEXT_LD_US(vle64_v, int64_t, lde_d_tlb, lde_d_host)
567
568	#define GEN_VEXT_ST_US(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST) \
569	void HELPER(NAME##_mask)(void vd, void v0, target_ulong base, \
570	CPURISCVState *env, uint32_t desc) \
571	{ \
572	uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \
573	vext_ldst_stride(vd, v0, base, stride, env, desc, false, \
574	STORE_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \
575	} \
576	\
577	void HELPER(NAME)(void vd, void v0, target_ulong base, \
578	CPURISCVState *env, uint32_t desc) \
579	{ \
580	vext_ldst_us(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
581	ctzl(sizeof(ETYPE)), env->vl, GETPC(), false); \
582	}
583
584	GEN_VEXT_ST_US(vse8_v, int8_t, ste_b_tlb, ste_b_host)
585	GEN_VEXT_ST_US(vse16_v, int16_t, ste_h_tlb, ste_h_host)
586	GEN_VEXT_ST_US(vse32_v, int32_t, ste_w_tlb, ste_w_host)
587	GEN_VEXT_ST_US(vse64_v, int64_t, ste_d_tlb, ste_d_host)
588
589	/*
590	* unit stride mask load and store, EEW = 1
591	*/
592	void HELPER(vlm_v)(void vd, void v0, target_ulong base,
593	CPURISCVState *env, uint32_t desc)
594	{
595	/* evl = ceil(vl/8) */
596	uint8_t evl = (env->vl + 7) >> 3;
597	vext_ldst_us(vd, base, env, desc, lde_b_tlb, lde_b_host,
598	0, evl, GETPC(), true);
599	}
600
601	void HELPER(vsm_v)(void vd, void v0, target_ulong base,
602	CPURISCVState *env, uint32_t desc)
603	{
604	/* evl = ceil(vl/8) */
605	uint8_t evl = (env->vl + 7) >> 3;
606	vext_ldst_us(vd, base, env, desc, ste_b_tlb, ste_b_host,
607	0, evl, GETPC(), false);
608	}
609
610	/*
611	* index: access vector element from indexed memory
612	*/
613	typedef target_ulong vext_get_index_addr(target_ulong base,
614	uint32_t idx, void *vs2);
615
616	#define GEN_VEXT_GET_INDEX_ADDR(NAME, ETYPE, H) \
617	static target_ulong NAME(target_ulong base, \
618	uint32_t idx, void *vs2) \
619	{ \
620	return (base + ((ETYPE )vs2 + H(idx))); \
621	}
622
623	GEN_VEXT_GET_INDEX_ADDR(idx_b, uint8_t, H1)
624	GEN_VEXT_GET_INDEX_ADDR(idx_h, uint16_t, H2)
625	GEN_VEXT_GET_INDEX_ADDR(idx_w, uint32_t, H4)
626	GEN_VEXT_GET_INDEX_ADDR(idx_d, uint64_t, H8)
627
628	static inline void
629	vext_ldst_index(void vd, void v0, target_ulong base,
630	void vs2, CPURISCVState env, uint32_t desc,
631	vext_get_index_addr get_index_addr,
632	vext_ldst_elem_fn_tlb *ldst_elem,
633	uint32_t log2_esz, uintptr_t ra)
634	{
635	uint32_t i, k;
636	uint32_t nf = vext_nf(desc);
637	uint32_t vm = vext_vm(desc);
638	uint32_t max_elems = vext_max_elems(desc, log2_esz);
639	uint32_t esz = 1 << log2_esz;
640	uint32_t vma = vext_vma(desc);
641
642	VSTART_CHECK_EARLY_EXIT(env, env->vl);
643
644	/* load bytes from guest memory */
645	for (i = env->vstart; i < env->vl; env->vstart = ++i) {
646	k = 0;
647	while (k < nf) {
648	if (!vm && !vext_elem_mask(v0, i)) {
649	/* set masked-off elements to 1s */
650	vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
651	(i + k * max_elems + 1) * esz);
652	k++;
653	continue;
654	}
655	abi_ptr addr = get_index_addr(base, i, vs2) + (k << log2_esz);
656	ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
657	k++;
658	}
659	}
660	env->vstart = 0;
661
662	vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
663	}
664
665	#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \
666	void HELPER(NAME)(void vd, void v0, target_ulong base, \
667	void vs2, CPURISCVState env, uint32_t desc) \
668	{ \
669	vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \
670	LOAD_FN, ctzl(sizeof(ETYPE)), GETPC()); \
671	}
672
673	GEN_VEXT_LD_INDEX(vlxei8_8_v, int8_t, idx_b, lde_b_tlb)
674	GEN_VEXT_LD_INDEX(vlxei8_16_v, int16_t, idx_b, lde_h_tlb)
675	GEN_VEXT_LD_INDEX(vlxei8_32_v, int32_t, idx_b, lde_w_tlb)
676	GEN_VEXT_LD_INDEX(vlxei8_64_v, int64_t, idx_b, lde_d_tlb)
677	GEN_VEXT_LD_INDEX(vlxei16_8_v, int8_t, idx_h, lde_b_tlb)
678	GEN_VEXT_LD_INDEX(vlxei16_16_v, int16_t, idx_h, lde_h_tlb)
679	GEN_VEXT_LD_INDEX(vlxei16_32_v, int32_t, idx_h, lde_w_tlb)
680	GEN_VEXT_LD_INDEX(vlxei16_64_v, int64_t, idx_h, lde_d_tlb)
681	GEN_VEXT_LD_INDEX(vlxei32_8_v, int8_t, idx_w, lde_b_tlb)
682	GEN_VEXT_LD_INDEX(vlxei32_16_v, int16_t, idx_w, lde_h_tlb)
683	GEN_VEXT_LD_INDEX(vlxei32_32_v, int32_t, idx_w, lde_w_tlb)
684	GEN_VEXT_LD_INDEX(vlxei32_64_v, int64_t, idx_w, lde_d_tlb)
685	GEN_VEXT_LD_INDEX(vlxei64_8_v, int8_t, idx_d, lde_b_tlb)
686	GEN_VEXT_LD_INDEX(vlxei64_16_v, int16_t, idx_d, lde_h_tlb)
687	GEN_VEXT_LD_INDEX(vlxei64_32_v, int32_t, idx_d, lde_w_tlb)
688	GEN_VEXT_LD_INDEX(vlxei64_64_v, int64_t, idx_d, lde_d_tlb)
689
690	#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN) \
691	void HELPER(NAME)(void vd, void v0, target_ulong base, \
692	void vs2, CPURISCVState env, uint32_t desc) \
693	{ \
694	vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \
695	STORE_FN, ctzl(sizeof(ETYPE)), \
696	GETPC()); \
697	}
698
699	GEN_VEXT_ST_INDEX(vsxei8_8_v, int8_t, idx_b, ste_b_tlb)
700	GEN_VEXT_ST_INDEX(vsxei8_16_v, int16_t, idx_b, ste_h_tlb)
701	GEN_VEXT_ST_INDEX(vsxei8_32_v, int32_t, idx_b, ste_w_tlb)
702	GEN_VEXT_ST_INDEX(vsxei8_64_v, int64_t, idx_b, ste_d_tlb)
703	GEN_VEXT_ST_INDEX(vsxei16_8_v, int8_t, idx_h, ste_b_tlb)
704	GEN_VEXT_ST_INDEX(vsxei16_16_v, int16_t, idx_h, ste_h_tlb)
705	GEN_VEXT_ST_INDEX(vsxei16_32_v, int32_t, idx_h, ste_w_tlb)
706	GEN_VEXT_ST_INDEX(vsxei16_64_v, int64_t, idx_h, ste_d_tlb)
707	GEN_VEXT_ST_INDEX(vsxei32_8_v, int8_t, idx_w, ste_b_tlb)
708	GEN_VEXT_ST_INDEX(vsxei32_16_v, int16_t, idx_w, ste_h_tlb)
709	GEN_VEXT_ST_INDEX(vsxei32_32_v, int32_t, idx_w, ste_w_tlb)
710	GEN_VEXT_ST_INDEX(vsxei32_64_v, int64_t, idx_w, ste_d_tlb)
711	GEN_VEXT_ST_INDEX(vsxei64_8_v, int8_t, idx_d, ste_b_tlb)
712	GEN_VEXT_ST_INDEX(vsxei64_16_v, int16_t, idx_d, ste_h_tlb)
713	GEN_VEXT_ST_INDEX(vsxei64_32_v, int32_t, idx_d, ste_w_tlb)
714	GEN_VEXT_ST_INDEX(vsxei64_64_v, int64_t, idx_d, ste_d_tlb)
715
716	/*
717	* unit-stride fault-only-fisrt load instructions
718	*/
719	static inline void
720	vext_ldff(void vd, void v0, target_ulong base, CPURISCVState *env,
721	uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
722	vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
723	{
724	uint32_t i, k, vl = 0;
725	uint32_t nf = vext_nf(desc);
726	uint32_t vm = vext_vm(desc);
727	uint32_t max_elems = vext_max_elems(desc, log2_esz);
728	uint32_t esz = 1 << log2_esz;
729	uint32_t msize = nf * esz;
730	uint32_t vma = vext_vma(desc);
731	target_ulong addr, addr_i, offset, remain, page_split, elems;
732	int mmu_index = riscv_env_mmu_index(env, false);
733	int flags;
734	void *host;
735
736	VSTART_CHECK_EARLY_EXIT(env, env->vl);
737
738	addr = base + ((env->vstart * nf) << log2_esz);
739	page_split = -(addr \| TARGET_PAGE_MASK);
740	/* Get number of elements */
741	elems = page_split / msize;
742	if (unlikely(env->vstart + elems >= env->vl)) {
743	elems = env->vl - env->vstart;
744	}
745
746	/* Check page permission/pmp/watchpoint/etc. */
747	probe_pages(env, addr, (env->vl - env->vstart) * msize, ra, MMU_DATA_LOAD,
748	mmu_index, &host, &flags, true);
749
750	if (flags & ~TLB_WATCHPOINT) {
751	/* probe every access */
752	for (i = env->vstart; i < env->vl; i++) {
753	if (!vm && !vext_elem_mask(v0, i)) {
754	continue;
755	}
756	addr_i = adjust_addr(env, base + i * (nf << log2_esz));
757	if (i == 0) {
758	/* Allow fault on first element. */
759	probe_pages(env, addr_i, nf << log2_esz, ra, MMU_DATA_LOAD,
760	mmu_index, &host, NULL, false);
761	} else {
762	remain = nf << log2_esz;
763	while (remain > 0) {
764	offset = -(addr_i \| TARGET_PAGE_MASK);
765
766	/* Probe nonfault on subsequent elements. */
767	probe_pages(env, addr_i, offset, 0, MMU_DATA_LOAD,
768	mmu_index, &host, &flags, true);
769
770	/*
771	* Stop if invalid (unmapped) or mmio (transaction may
772	* fail). Do not stop if watchpoint, as the spec says that
773	* first-fault should continue to access the same
774	* elements regardless of any watchpoint.
775	*/
776	if (flags & ~TLB_WATCHPOINT) {
777	vl = i;
778	goto ProbeSuccess;
779	}
780	if (remain <= offset) {
781	break;
782	}
783	remain -= offset;
784	addr_i = adjust_addr(env, addr_i + offset);
785	}
786	}
787	}
788	}
789	ProbeSuccess:
790	/* load bytes from guest memory */
791	if (vl != 0) {
792	env->vl = vl;
793	}
794
795	if (env->vstart < env->vl) {
796	if (vm) {
797	/* Load/store elements in the first page */
798	if (likely(elems)) {
799	vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
800	log2_esz, true, mmu_index, ldst_tlb,
801	ldst_host, ra);
802	}
803
804	/* Load/store elements in the second page */
805	if (unlikely(env->vstart < env->vl)) {
806	/* Cross page element */
807	if (unlikely(page_split % msize)) {
808	for (k = 0; k < nf; k++) {
809	addr = base + ((env->vstart * nf + k) << log2_esz);
810	ldst_tlb(env, adjust_addr(env, addr),
811	env->vstart + k * max_elems, vd, ra);
812	}
813	env->vstart++;
814	}
815
816	addr = base + ((env->vstart * nf) << log2_esz);
817	/* Get number of elements of second page */
818	elems = env->vl - env->vstart;
819
820	/* Load/store elements in the second page */
821	vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
822	log2_esz, true, mmu_index, ldst_tlb,
823	ldst_host, ra);
824	}
825	} else {
826	for (i = env->vstart; i < env->vl; i++) {
827	k = 0;
828	while (k < nf) {
829	if (!vext_elem_mask(v0, i)) {
830	/* set masked-off elements to 1s */
831	vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
832	(i + k * max_elems + 1) * esz);
833	k++;
834	continue;
835	}
836	addr = base + ((i * nf + k) << log2_esz);
837	ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems,
838	vd, ra);
839	k++;
840	}
841	}
842	}
843	}
844	env->vstart = 0;
845
846	vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems);
847	}
848
849	#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
850	void HELPER(NAME)(void vd, void v0, target_ulong base, \
851	CPURISCVState *env, uint32_t desc) \
852	{ \
853	vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB, \
854	LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC()); \
855	}
856
857	GEN_VEXT_LDFF(vle8ff_v, int8_t, lde_b_tlb, lde_b_host)
858	GEN_VEXT_LDFF(vle16ff_v, int16_t, lde_h_tlb, lde_h_host)
859	GEN_VEXT_LDFF(vle32ff_v, int32_t, lde_w_tlb, lde_w_host)
860	GEN_VEXT_LDFF(vle64ff_v, int64_t, lde_d_tlb, lde_d_host)
861
862	#define DO_SWAP(N, M) (M)
863	#define DO_AND(N, M) (N & M)
864	#define DO_XOR(N, M) (N ^ M)
865	#define DO_OR(N, M) (N \| M)
866	#define DO_ADD(N, M) (N + M)
867
868	/* Signed min/max */
869	#define DO_MAX(N, M) ((N) >= (M) ? (N) : (M))
870	#define DO_MIN(N, M) ((N) >= (M) ? (M) : (N))
871
872	/*
873	* load and store whole register instructions
874	*/
875	static inline QEMU_ALWAYS_INLINE void
876	vext_ldst_whole(void vd, target_ulong base, CPURISCVState env, uint32_t desc,
877	vext_ldst_elem_fn_tlb *ldst_tlb,
878	vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
879	uintptr_t ra, bool is_load)
880	{
881	target_ulong page_split, elems, addr;
882	uint32_t nf = vext_nf(desc);
883	uint32_t vlenb = riscv_cpu_cfg(env)->vlenb;
884	uint32_t max_elems = vlenb >> log2_esz;
885	uint32_t evl = nf * max_elems;
886	uint32_t esz = 1 << log2_esz;
887	int mmu_index = riscv_env_mmu_index(env, false);
888
889	/* Calculate the page range of first page */
890	addr = base + (env->vstart << log2_esz);
891	page_split = -(addr \| TARGET_PAGE_MASK);
892	/* Get number of elements */
893	elems = page_split / esz;
894	if (unlikely(env->vstart + elems >= evl)) {
895	elems = evl - env->vstart;
896	}
897
898	/* Load/store elements in the first page */
899	if (likely(elems)) {
900	vext_page_ldst_us(env, vd, addr, elems, 1, max_elems, log2_esz,
901	is_load, mmu_index, ldst_tlb, ldst_host, ra);
902	}
903
904	/* Load/store elements in the second page */
905	if (unlikely(env->vstart < evl)) {
906	/* Cross page element */
907	if (unlikely(page_split % esz)) {
908	addr = base + (env->vstart << log2_esz);
909	ldst_tlb(env, adjust_addr(env, addr), env->vstart, vd, ra);
910	env->vstart++;
911	}
912
913	addr = base + (env->vstart << log2_esz);
914	/* Get number of elements of second page */
915	elems = evl - env->vstart;
916
917	/* Load/store elements in the second page */
918	vext_page_ldst_us(env, vd, addr, elems, 1, max_elems, log2_esz,
919	is_load, mmu_index, ldst_tlb, ldst_host, ra);
920	}
921
922	env->vstart = 0;
923	}
924
925	#define GEN_VEXT_LD_WHOLE(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \
926	void HELPER(NAME)(void vd, target_ulong base, CPURISCVState env, \
927	uint32_t desc) \
928	{ \
929	vext_ldst_whole(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \
930	ctzl(sizeof(ETYPE)), GETPC(), true); \
931	}
932
933	GEN_VEXT_LD_WHOLE(vl1re8_v, int8_t, lde_b_tlb, lde_b_host)
934	GEN_VEXT_LD_WHOLE(vl1re16_v, int16_t, lde_h_tlb, lde_h_host)
935	GEN_VEXT_LD_WHOLE(vl1re32_v, int32_t, lde_w_tlb, lde_w_host)
936	GEN_VEXT_LD_WHOLE(vl1re64_v, int64_t, lde_d_tlb, lde_d_host)
937	GEN_VEXT_LD_WHOLE(vl2re8_v, int8_t, lde_b_tlb, lde_b_host)
938	GEN_VEXT_LD_WHOLE(vl2re16_v, int16_t, lde_h_tlb, lde_h_host)
939	GEN_VEXT_LD_WHOLE(vl2re32_v, int32_t, lde_w_tlb, lde_w_host)
940	GEN_VEXT_LD_WHOLE(vl2re64_v, int64_t, lde_d_tlb, lde_d_host)
941	GEN_VEXT_LD_WHOLE(vl4re8_v, int8_t, lde_b_tlb, lde_b_host)
942	GEN_VEXT_LD_WHOLE(vl4re16_v, int16_t, lde_h_tlb, lde_h_host)
943	GEN_VEXT_LD_WHOLE(vl4re32_v, int32_t, lde_w_tlb, lde_w_host)
944	GEN_VEXT_LD_WHOLE(vl4re64_v, int64_t, lde_d_tlb, lde_d_host)
945	GEN_VEXT_LD_WHOLE(vl8re8_v, int8_t, lde_b_tlb, lde_b_host)
946	GEN_VEXT_LD_WHOLE(vl8re16_v, int16_t, lde_h_tlb, lde_h_host)
947	GEN_VEXT_LD_WHOLE(vl8re32_v, int32_t, lde_w_tlb, lde_w_host)
948	GEN_VEXT_LD_WHOLE(vl8re64_v, int64_t, lde_d_tlb, lde_d_host)
949
950	#define GEN_VEXT_ST_WHOLE(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST) \
951	void HELPER(NAME)(void vd, target_ulong base, CPURISCVState env, \
952	uint32_t desc) \
953	{ \
954	vext_ldst_whole(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \
955	ctzl(sizeof(ETYPE)), GETPC(), false); \
956	}
957
958	GEN_VEXT_ST_WHOLE(vs1r_v, int8_t, ste_b_tlb, ste_b_host)
959	GEN_VEXT_ST_WHOLE(vs2r_v, int8_t, ste_b_tlb, ste_b_host)
960	GEN_VEXT_ST_WHOLE(vs4r_v, int8_t, ste_b_tlb, ste_b_host)
961	GEN_VEXT_ST_WHOLE(vs8r_v, int8_t, ste_b_tlb, ste_b_host)
962
963	/*
964	* Vector Integer Arithmetic Instructions
965	*/
966
967	/* (TD, T1, T2, TX1, TX2) */
968	#define OP_SSS_B int8_t, int8_t, int8_t, int8_t, int8_t
969	#define OP_SSS_H int16_t, int16_t, int16_t, int16_t, int16_t
970	#define OP_SSS_W int32_t, int32_t, int32_t, int32_t, int32_t
971	#define OP_SSS_D int64_t, int64_t, int64_t, int64_t, int64_t
972	#define OP_SUS_B int8_t, uint8_t, int8_t, uint8_t, int8_t
973	#define OP_SUS_H int16_t, uint16_t, int16_t, uint16_t, int16_t
974	#define OP_SUS_W int32_t, uint32_t, int32_t, uint32_t, int32_t
975	#define OP_SUS_D int64_t, uint64_t, int64_t, uint64_t, int64_t
976	#define WOP_SSS_B int16_t, int8_t, int8_t, int16_t, int16_t
977	#define WOP_SSS_H int32_t, int16_t, int16_t, int32_t, int32_t
978	#define WOP_SSS_W int64_t, int32_t, int32_t, int64_t, int64_t
979	#define WOP_SUS_B int16_t, uint8_t, int8_t, uint16_t, int16_t
980	#define WOP_SUS_H int32_t, uint16_t, int16_t, uint32_t, int32_t
981	#define WOP_SUS_W int64_t, uint32_t, int32_t, uint64_t, int64_t
982	#define WOP_SSU_B int16_t, int8_t, uint8_t, int16_t, uint16_t
983	#define WOP_SSU_H int32_t, int16_t, uint16_t, int32_t, uint32_t
984	#define WOP_SSU_W int64_t, int32_t, uint32_t, int64_t, uint64_t
985	#define NOP_SSS_B int8_t, int8_t, int16_t, int8_t, int16_t
986	#define NOP_SSS_H int16_t, int16_t, int32_t, int16_t, int32_t
987	#define NOP_SSS_W int32_t, int32_t, int64_t, int32_t, int64_t
988	#define NOP_UUU_B uint8_t, uint8_t, uint16_t, uint8_t, uint16_t
989	#define NOP_UUU_H uint16_t, uint16_t, uint32_t, uint16_t, uint32_t
990	#define NOP_UUU_W uint32_t, uint32_t, uint64_t, uint32_t, uint64_t
991
992	#define DO_SUB(N, M) (N - M)
993	#define DO_RSUB(N, M) (M - N)
994
995	RVVCALL(OPIVV2, vadd_vv_b, OP_SSS_B, H1, H1, H1, DO_ADD)
996	RVVCALL(OPIVV2, vadd_vv_h, OP_SSS_H, H2, H2, H2, DO_ADD)
997	RVVCALL(OPIVV2, vadd_vv_w, OP_SSS_W, H4, H4, H4, DO_ADD)
998	RVVCALL(OPIVV2, vadd_vv_d, OP_SSS_D, H8, H8, H8, DO_ADD)
999	RVVCALL(OPIVV2, vsub_vv_b, OP_SSS_B, H1, H1, H1, DO_SUB)
1000	RVVCALL(OPIVV2, vsub_vv_h, OP_SSS_H, H2, H2, H2, DO_SUB)
1001	RVVCALL(OPIVV2, vsub_vv_w, OP_SSS_W, H4, H4, H4, DO_SUB)
1002	RVVCALL(OPIVV2, vsub_vv_d, OP_SSS_D, H8, H8, H8, DO_SUB)
1003
1004	GEN_VEXT_VV(vadd_vv_b, 1)
1005	GEN_VEXT_VV(vadd_vv_h, 2)
1006	GEN_VEXT_VV(vadd_vv_w, 4)
1007	GEN_VEXT_VV(vadd_vv_d, 8)
1008	GEN_VEXT_VV(vsub_vv_b, 1)
1009	GEN_VEXT_VV(vsub_vv_h, 2)
1010	GEN_VEXT_VV(vsub_vv_w, 4)
1011	GEN_VEXT_VV(vsub_vv_d, 8)
1012
1013
1014	RVVCALL(OPIVX2, vadd_vx_b, OP_SSS_B, H1, H1, DO_ADD)
1015	RVVCALL(OPIVX2, vadd_vx_h, OP_SSS_H, H2, H2, DO_ADD)
1016	RVVCALL(OPIVX2, vadd_vx_w, OP_SSS_W, H4, H4, DO_ADD)
1017	RVVCALL(OPIVX2, vadd_vx_d, OP_SSS_D, H8, H8, DO_ADD)
1018	RVVCALL(OPIVX2, vsub_vx_b, OP_SSS_B, H1, H1, DO_SUB)
1019	RVVCALL(OPIVX2, vsub_vx_h, OP_SSS_H, H2, H2, DO_SUB)
1020	RVVCALL(OPIVX2, vsub_vx_w, OP_SSS_W, H4, H4, DO_SUB)
1021	RVVCALL(OPIVX2, vsub_vx_d, OP_SSS_D, H8, H8, DO_SUB)
1022	RVVCALL(OPIVX2, vrsub_vx_b, OP_SSS_B, H1, H1, DO_RSUB)
1023	RVVCALL(OPIVX2, vrsub_vx_h, OP_SSS_H, H2, H2, DO_RSUB)
1024	RVVCALL(OPIVX2, vrsub_vx_w, OP_SSS_W, H4, H4, DO_RSUB)
1025	RVVCALL(OPIVX2, vrsub_vx_d, OP_SSS_D, H8, H8, DO_RSUB)
1026
1027	GEN_VEXT_VX(vadd_vx_b, 1)
1028	GEN_VEXT_VX(vadd_vx_h, 2)
1029	GEN_VEXT_VX(vadd_vx_w, 4)
1030	GEN_VEXT_VX(vadd_vx_d, 8)
1031	GEN_VEXT_VX(vsub_vx_b, 1)
1032	GEN_VEXT_VX(vsub_vx_h, 2)
1033	GEN_VEXT_VX(vsub_vx_w, 4)
1034	GEN_VEXT_VX(vsub_vx_d, 8)
1035	GEN_VEXT_VX(vrsub_vx_b, 1)
1036	GEN_VEXT_VX(vrsub_vx_h, 2)
1037	GEN_VEXT_VX(vrsub_vx_w, 4)
1038	GEN_VEXT_VX(vrsub_vx_d, 8)
1039
1040	void HELPER(vec_rsubs8)(void d, void a, uint64_t b, uint32_t desc)
1041	{
1042	intptr_t oprsz = simd_oprsz(desc);
1043	intptr_t i;
1044
1045	for (i = 0; i < oprsz; i += sizeof(uint8_t)) {
1046	(uint8_t )(d + i) = (uint8_t)b - (uint8_t )(a + i);
1047	}
1048	}
1049
1050	void HELPER(vec_rsubs16)(void d, void a, uint64_t b, uint32_t desc)
1051	{
1052	intptr_t oprsz = simd_oprsz(desc);
1053	intptr_t i;
1054
1055	for (i = 0; i < oprsz; i += sizeof(uint16_t)) {
1056	(uint16_t )(d + i) = (uint16_t)b - (uint16_t )(a + i);
1057	}
1058	}
1059
1060	void HELPER(vec_rsubs32)(void d, void a, uint64_t b, uint32_t desc)
1061	{
1062	intptr_t oprsz = simd_oprsz(desc);
1063	intptr_t i;
1064
1065	for (i = 0; i < oprsz; i += sizeof(uint32_t)) {
1066	(uint32_t )(d + i) = (uint32_t)b - (uint32_t )(a + i);
1067	}
1068	}
1069
1070	void HELPER(vec_rsubs64)(void d, void a, uint64_t b, uint32_t desc)
1071	{
1072	intptr_t oprsz = simd_oprsz(desc);
1073	intptr_t i;
1074
1075	for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
1076	(uint64_t )(d + i) = b - (uint64_t )(a + i);
1077	}
1078	}
1079
1080	/* Vector Widening Integer Add/Subtract */
1081	#define WOP_UUU_B uint16_t, uint8_t, uint8_t, uint16_t, uint16_t
1082	#define WOP_UUU_H uint32_t, uint16_t, uint16_t, uint32_t, uint32_t
1083	#define WOP_UUU_W uint64_t, uint32_t, uint32_t, uint64_t, uint64_t
1084	#define WOP_SSS_B int16_t, int8_t, int8_t, int16_t, int16_t
1085	#define WOP_SSS_H int32_t, int16_t, int16_t, int32_t, int32_t
1086	#define WOP_SSS_W int64_t, int32_t, int32_t, int64_t, int64_t
1087	#define WOP_WUUU_B uint16_t, uint8_t, uint16_t, uint16_t, uint16_t
1088	#define WOP_WUUU_H uint32_t, uint16_t, uint32_t, uint32_t, uint32_t
1089	#define WOP_WUUU_W uint64_t, uint32_t, uint64_t, uint64_t, uint64_t
1090	#define WOP_WSSS_B int16_t, int8_t, int16_t, int16_t, int16_t
1091	#define WOP_WSSS_H int32_t, int16_t, int32_t, int32_t, int32_t
1092	#define WOP_WSSS_W int64_t, int32_t, int64_t, int64_t, int64_t
1093	RVVCALL(OPIVV2, vwaddu_vv_b, WOP_UUU_B, H2, H1, H1, DO_ADD)
1094	RVVCALL(OPIVV2, vwaddu_vv_h, WOP_UUU_H, H4, H2, H2, DO_ADD)
1095	RVVCALL(OPIVV2, vwaddu_vv_w, WOP_UUU_W, H8, H4, H4, DO_ADD)
1096	RVVCALL(OPIVV2, vwsubu_vv_b, WOP_UUU_B, H2, H1, H1, DO_SUB)
1097	RVVCALL(OPIVV2, vwsubu_vv_h, WOP_UUU_H, H4, H2, H2, DO_SUB)
1098	RVVCALL(OPIVV2, vwsubu_vv_w, WOP_UUU_W, H8, H4, H4, DO_SUB)
1099	RVVCALL(OPIVV2, vwadd_vv_b, WOP_SSS_B, H2, H1, H1, DO_ADD)
1100	RVVCALL(OPIVV2, vwadd_vv_h, WOP_SSS_H, H4, H2, H2, DO_ADD)
1101	RVVCALL(OPIVV2, vwadd_vv_w, WOP_SSS_W, H8, H4, H4, DO_ADD)
1102	RVVCALL(OPIVV2, vwsub_vv_b, WOP_SSS_B, H2, H1, H1, DO_SUB)
1103	RVVCALL(OPIVV2, vwsub_vv_h, WOP_SSS_H, H4, H2, H2, DO_SUB)
1104	RVVCALL(OPIVV2, vwsub_vv_w, WOP_SSS_W, H8, H4, H4, DO_SUB)
1105	RVVCALL(OPIVV2, vwaddu_wv_b, WOP_WUUU_B, H2, H1, H1, DO_ADD)
1106	RVVCALL(OPIVV2, vwaddu_wv_h, WOP_WUUU_H, H4, H2, H2, DO_ADD)
1107	RVVCALL(OPIVV2, vwaddu_wv_w, WOP_WUUU_W, H8, H4, H4, DO_ADD)
1108	RVVCALL(OPIVV2, vwsubu_wv_b, WOP_WUUU_B, H2, H1, H1, DO_SUB)
1109	RVVCALL(OPIVV2, vwsubu_wv_h, WOP_WUUU_H, H4, H2, H2, DO_SUB)
1110	RVVCALL(OPIVV2, vwsubu_wv_w, WOP_WUUU_W, H8, H4, H4, DO_SUB)
1111	RVVCALL(OPIVV2, vwadd_wv_b, WOP_WSSS_B, H2, H1, H1, DO_ADD)
1112	RVVCALL(OPIVV2, vwadd_wv_h, WOP_WSSS_H, H4, H2, H2, DO_ADD)
1113	RVVCALL(OPIVV2, vwadd_wv_w, WOP_WSSS_W, H8, H4, H4, DO_ADD)
1114	RVVCALL(OPIVV2, vwsub_wv_b, WOP_WSSS_B, H2, H1, H1, DO_SUB)
1115	RVVCALL(OPIVV2, vwsub_wv_h, WOP_WSSS_H, H4, H2, H2, DO_SUB)
1116	RVVCALL(OPIVV2, vwsub_wv_w, WOP_WSSS_W, H8, H4, H4, DO_SUB)
1117	GEN_VEXT_VV(vwaddu_vv_b, 2)
1118	GEN_VEXT_VV(vwaddu_vv_h, 4)
1119	GEN_VEXT_VV(vwaddu_vv_w, 8)
1120	GEN_VEXT_VV(vwsubu_vv_b, 2)
1121	GEN_VEXT_VV(vwsubu_vv_h, 4)
1122	GEN_VEXT_VV(vwsubu_vv_w, 8)
1123	GEN_VEXT_VV(vwadd_vv_b, 2)
1124	GEN_VEXT_VV(vwadd_vv_h, 4)
1125	GEN_VEXT_VV(vwadd_vv_w, 8)
1126	GEN_VEXT_VV(vwsub_vv_b, 2)
1127	GEN_VEXT_VV(vwsub_vv_h, 4)
1128	GEN_VEXT_VV(vwsub_vv_w, 8)
1129	GEN_VEXT_VV(vwaddu_wv_b, 2)
1130	GEN_VEXT_VV(vwaddu_wv_h, 4)
1131	GEN_VEXT_VV(vwaddu_wv_w, 8)
1132	GEN_VEXT_VV(vwsubu_wv_b, 2)
1133	GEN_VEXT_VV(vwsubu_wv_h, 4)
1134	GEN_VEXT_VV(vwsubu_wv_w, 8)
1135	GEN_VEXT_VV(vwadd_wv_b, 2)
1136	GEN_VEXT_VV(vwadd_wv_h, 4)
1137	GEN_VEXT_VV(vwadd_wv_w, 8)
1138	GEN_VEXT_VV(vwsub_wv_b, 2)
1139	GEN_VEXT_VV(vwsub_wv_h, 4)
1140	GEN_VEXT_VV(vwsub_wv_w, 8)
1141
1142	RVVCALL(OPIVX2, vwaddu_vx_b, WOP_UUU_B, H2, H1, DO_ADD)
1143	RVVCALL(OPIVX2, vwaddu_vx_h, WOP_UUU_H, H4, H2, DO_ADD)
1144	RVVCALL(OPIVX2, vwaddu_vx_w, WOP_UUU_W, H8, H4, DO_ADD)
1145	RVVCALL(OPIVX2, vwsubu_vx_b, WOP_UUU_B, H2, H1, DO_SUB)
1146	RVVCALL(OPIVX2, vwsubu_vx_h, WOP_UUU_H, H4, H2, DO_SUB)
1147	RVVCALL(OPIVX2, vwsubu_vx_w, WOP_UUU_W, H8, H4, DO_SUB)
1148	RVVCALL(OPIVX2, vwadd_vx_b, WOP_SSS_B, H2, H1, DO_ADD)
1149	RVVCALL(OPIVX2, vwadd_vx_h, WOP_SSS_H, H4, H2, DO_ADD)
1150	RVVCALL(OPIVX2, vwadd_vx_w, WOP_SSS_W, H8, H4, DO_ADD)
1151	RVVCALL(OPIVX2, vwsub_vx_b, WOP_SSS_B, H2, H1, DO_SUB)
1152	RVVCALL(OPIVX2, vwsub_vx_h, WOP_SSS_H, H4, H2, DO_SUB)
1153	RVVCALL(OPIVX2, vwsub_vx_w, WOP_SSS_W, H8, H4, DO_SUB)
1154	RVVCALL(OPIVX2, vwaddu_wx_b, WOP_WUUU_B, H2, H1, DO_ADD)
1155	RVVCALL(OPIVX2, vwaddu_wx_h, WOP_WUUU_H, H4, H2, DO_ADD)
1156	RVVCALL(OPIVX2, vwaddu_wx_w, WOP_WUUU_W, H8, H4, DO_ADD)
1157	RVVCALL(OPIVX2, vwsubu_wx_b, WOP_WUUU_B, H2, H1, DO_SUB)
1158	RVVCALL(OPIVX2, vwsubu_wx_h, WOP_WUUU_H, H4, H2, DO_SUB)
1159	RVVCALL(OPIVX2, vwsubu_wx_w, WOP_WUUU_W, H8, H4, DO_SUB)
1160	RVVCALL(OPIVX2, vwadd_wx_b, WOP_WSSS_B, H2, H1, DO_ADD)
1161	RVVCALL(OPIVX2, vwadd_wx_h, WOP_WSSS_H, H4, H2, DO_ADD)
1162	RVVCALL(OPIVX2, vwadd_wx_w, WOP_WSSS_W, H8, H4, DO_ADD)
1163	RVVCALL(OPIVX2, vwsub_wx_b, WOP_WSSS_B, H2, H1, DO_SUB)
1164	RVVCALL(OPIVX2, vwsub_wx_h, WOP_WSSS_H, H4, H2, DO_SUB)
1165	RVVCALL(OPIVX2, vwsub_wx_w, WOP_WSSS_W, H8, H4, DO_SUB)
1166	GEN_VEXT_VX(vwaddu_vx_b, 2)
1167	GEN_VEXT_VX(vwaddu_vx_h, 4)
1168	GEN_VEXT_VX(vwaddu_vx_w, 8)
1169	GEN_VEXT_VX(vwsubu_vx_b, 2)
1170	GEN_VEXT_VX(vwsubu_vx_h, 4)
1171	GEN_VEXT_VX(vwsubu_vx_w, 8)
1172	GEN_VEXT_VX(vwadd_vx_b, 2)
1173	GEN_VEXT_VX(vwadd_vx_h, 4)
1174	GEN_VEXT_VX(vwadd_vx_w, 8)
1175	GEN_VEXT_VX(vwsub_vx_b, 2)
1176	GEN_VEXT_VX(vwsub_vx_h, 4)
1177	GEN_VEXT_VX(vwsub_vx_w, 8)
1178	GEN_VEXT_VX(vwaddu_wx_b, 2)
1179	GEN_VEXT_VX(vwaddu_wx_h, 4)
1180	GEN_VEXT_VX(vwaddu_wx_w, 8)
1181	GEN_VEXT_VX(vwsubu_wx_b, 2)
1182	GEN_VEXT_VX(vwsubu_wx_h, 4)
1183	GEN_VEXT_VX(vwsubu_wx_w, 8)
1184	GEN_VEXT_VX(vwadd_wx_b, 2)
1185	GEN_VEXT_VX(vwadd_wx_h, 4)
1186	GEN_VEXT_VX(vwadd_wx_w, 8)
1187	GEN_VEXT_VX(vwsub_wx_b, 2)
1188	GEN_VEXT_VX(vwsub_wx_h, 4)
1189	GEN_VEXT_VX(vwsub_wx_w, 8)
1190
1191	/* Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions */
1192	#define DO_VADC(N, M, C) (N + M + C)
1193	#define DO_VSBC(N, M, C) (N - M - C)
1194
1195	#define GEN_VEXT_VADC_VVM(NAME, ETYPE, H, DO_OP) \
1196	void HELPER(NAME)(void vd, void v0, void vs1, void vs2, \
1197	CPURISCVState *env, uint32_t desc) \
1198	{ \
1199	uint32_t vl = env->vl; \
1200	uint32_t esz = sizeof(ETYPE); \
1201	uint32_t total_elems = \
1202	vext_get_total_elems(env, desc, esz); \
1203	uint32_t vta = vext_vta(desc); \
1204	uint32_t i; \
1205	\
1206	VSTART_CHECK_EARLY_EXIT(env, vl); \
1207	\
1208	for (i = env->vstart; i < vl; i++) { \
1209	ETYPE s1 = ((ETYPE )vs1 + H(i)); \
1210	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
1211	ETYPE carry = vext_elem_mask(v0, i); \
1212	\
1213	((ETYPE )vd + H(i)) = DO_OP(s2, s1, carry); \
1214	} \
1215	env->vstart = 0; \
1216	/* set tail elements to 1s */ \
1217	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
1218	}
1219
1220	GEN_VEXT_VADC_VVM(vadc_vvm_b, uint8_t, H1, DO_VADC)
1221	GEN_VEXT_VADC_VVM(vadc_vvm_h, uint16_t, H2, DO_VADC)
1222	GEN_VEXT_VADC_VVM(vadc_vvm_w, uint32_t, H4, DO_VADC)
1223	GEN_VEXT_VADC_VVM(vadc_vvm_d, uint64_t, H8, DO_VADC)
1224
1225	GEN_VEXT_VADC_VVM(vsbc_vvm_b, uint8_t, H1, DO_VSBC)
1226	GEN_VEXT_VADC_VVM(vsbc_vvm_h, uint16_t, H2, DO_VSBC)
1227	GEN_VEXT_VADC_VVM(vsbc_vvm_w, uint32_t, H4, DO_VSBC)
1228	GEN_VEXT_VADC_VVM(vsbc_vvm_d, uint64_t, H8, DO_VSBC)
1229
1230	#define GEN_VEXT_VADC_VXM(NAME, ETYPE, H, DO_OP) \
1231	void HELPER(NAME)(void vd, void v0, target_ulong s1, void *vs2, \
1232	CPURISCVState *env, uint32_t desc) \
1233	{ \
1234	uint32_t vl = env->vl; \
1235	uint32_t esz = sizeof(ETYPE); \
1236	uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
1237	uint32_t vta = vext_vta(desc); \
1238	uint32_t i; \
1239	\
1240	VSTART_CHECK_EARLY_EXIT(env, vl); \
1241	\
1242	for (i = env->vstart; i < vl; i++) { \
1243	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
1244	ETYPE carry = vext_elem_mask(v0, i); \
1245	\
1246	((ETYPE )vd + H(i)) = DO_OP(s2, (ETYPE)(target_long)s1, carry);\
1247	} \
1248	env->vstart = 0; \
1249	/* set tail elements to 1s */ \
1250	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
1251	}
1252
1253	GEN_VEXT_VADC_VXM(vadc_vxm_b, uint8_t, H1, DO_VADC)
1254	GEN_VEXT_VADC_VXM(vadc_vxm_h, uint16_t, H2, DO_VADC)
1255	GEN_VEXT_VADC_VXM(vadc_vxm_w, uint32_t, H4, DO_VADC)
1256	GEN_VEXT_VADC_VXM(vadc_vxm_d, uint64_t, H8, DO_VADC)
1257
1258	GEN_VEXT_VADC_VXM(vsbc_vxm_b, uint8_t, H1, DO_VSBC)
1259	GEN_VEXT_VADC_VXM(vsbc_vxm_h, uint16_t, H2, DO_VSBC)
1260	GEN_VEXT_VADC_VXM(vsbc_vxm_w, uint32_t, H4, DO_VSBC)
1261	GEN_VEXT_VADC_VXM(vsbc_vxm_d, uint64_t, H8, DO_VSBC)
1262
1263	#define DO_MADC(N, M, C) (C ? (__typeof(N))(N + M + 1) <= N : \
1264	(__typeof(N))(N + M) < N)
1265	#define DO_MSBC(N, M, C) (C ? N <= M : N < M)
1266
1267	#define GEN_VEXT_VMADC_VVM(NAME, ETYPE, H, DO_OP) \
1268	void HELPER(NAME)(void vd, void v0, void vs1, void vs2, \
1269	CPURISCVState *env, uint32_t desc) \
1270	{ \
1271	uint32_t vl = env->vl; \
1272	uint32_t vm = vext_vm(desc); \
1273	uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \
1274	uint32_t vta_all_1s = vext_vta_all_1s(desc); \
1275	uint32_t i; \
1276	\
1277	VSTART_CHECK_EARLY_EXIT(env, vl); \
1278	\
1279	for (i = env->vstart; i < vl; i++) { \
1280	ETYPE s1 = ((ETYPE )vs1 + H(i)); \
1281	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
1282	ETYPE carry = !vm && vext_elem_mask(v0, i); \
1283	vext_set_elem_mask(vd, i, DO_OP(s2, s1, carry)); \
1284	} \
1285	env->vstart = 0; \
1286	/*
1287	* mask destination register are always tail-agnostic
1288	* set tail elements to 1s
1289	*/ \
1290	if (vta_all_1s) { \
1291	for (; i < total_elems; i++) { \
1292	vext_set_elem_mask(vd, i, 1); \
1293	} \
1294	} \
1295	}
1296
1297	GEN_VEXT_VMADC_VVM(vmadc_vvm_b, uint8_t, H1, DO_MADC)
1298	GEN_VEXT_VMADC_VVM(vmadc_vvm_h, uint16_t, H2, DO_MADC)
1299	GEN_VEXT_VMADC_VVM(vmadc_vvm_w, uint32_t, H4, DO_MADC)
1300	GEN_VEXT_VMADC_VVM(vmadc_vvm_d, uint64_t, H8, DO_MADC)
1301
1302	GEN_VEXT_VMADC_VVM(vmsbc_vvm_b, uint8_t, H1, DO_MSBC)
1303	GEN_VEXT_VMADC_VVM(vmsbc_vvm_h, uint16_t, H2, DO_MSBC)
1304	GEN_VEXT_VMADC_VVM(vmsbc_vvm_w, uint32_t, H4, DO_MSBC)
1305	GEN_VEXT_VMADC_VVM(vmsbc_vvm_d, uint64_t, H8, DO_MSBC)
1306
1307	#define GEN_VEXT_VMADC_VXM(NAME, ETYPE, H, DO_OP) \
1308	void HELPER(NAME)(void vd, void v0, target_ulong s1, \
1309	void vs2, CPURISCVState env, uint32_t desc) \
1310	{ \
1311	uint32_t vl = env->vl; \
1312	uint32_t vm = vext_vm(desc); \
1313	uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \
1314	uint32_t vta_all_1s = vext_vta_all_1s(desc); \
1315	uint32_t i; \
1316	\
1317	VSTART_CHECK_EARLY_EXIT(env, vl); \
1318	\
1319	for (i = env->vstart; i < vl; i++) { \
1320	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
1321	ETYPE carry = !vm && vext_elem_mask(v0, i); \
1322	vext_set_elem_mask(vd, i, \
1323	DO_OP(s2, (ETYPE)(target_long)s1, carry)); \
1324	} \
1325	env->vstart = 0; \
1326	/*
1327	* mask destination register are always tail-agnostic
1328	* set tail elements to 1s
1329	*/ \
1330	if (vta_all_1s) { \
1331	for (; i < total_elems; i++) { \
1332	vext_set_elem_mask(vd, i, 1); \
1333	} \
1334	} \
1335	}
1336
1337	GEN_VEXT_VMADC_VXM(vmadc_vxm_b, uint8_t, H1, DO_MADC)
1338	GEN_VEXT_VMADC_VXM(vmadc_vxm_h, uint16_t, H2, DO_MADC)
1339	GEN_VEXT_VMADC_VXM(vmadc_vxm_w, uint32_t, H4, DO_MADC)
1340	GEN_VEXT_VMADC_VXM(vmadc_vxm_d, uint64_t, H8, DO_MADC)
1341
1342	GEN_VEXT_VMADC_VXM(vmsbc_vxm_b, uint8_t, H1, DO_MSBC)
1343	GEN_VEXT_VMADC_VXM(vmsbc_vxm_h, uint16_t, H2, DO_MSBC)
1344	GEN_VEXT_VMADC_VXM(vmsbc_vxm_w, uint32_t, H4, DO_MSBC)
1345	GEN_VEXT_VMADC_VXM(vmsbc_vxm_d, uint64_t, H8, DO_MSBC)
1346
1347	/* Vector Bitwise Logical Instructions */
1348	RVVCALL(OPIVV2, vand_vv_b, OP_SSS_B, H1, H1, H1, DO_AND)
1349	RVVCALL(OPIVV2, vand_vv_h, OP_SSS_H, H2, H2, H2, DO_AND)
1350	RVVCALL(OPIVV2, vand_vv_w, OP_SSS_W, H4, H4, H4, DO_AND)
1351	RVVCALL(OPIVV2, vand_vv_d, OP_SSS_D, H8, H8, H8, DO_AND)
1352	RVVCALL(OPIVV2, vor_vv_b, OP_SSS_B, H1, H1, H1, DO_OR)
1353	RVVCALL(OPIVV2, vor_vv_h, OP_SSS_H, H2, H2, H2, DO_OR)
1354	RVVCALL(OPIVV2, vor_vv_w, OP_SSS_W, H4, H4, H4, DO_OR)
1355	RVVCALL(OPIVV2, vor_vv_d, OP_SSS_D, H8, H8, H8, DO_OR)
1356	RVVCALL(OPIVV2, vxor_vv_b, OP_SSS_B, H1, H1, H1, DO_XOR)
1357	RVVCALL(OPIVV2, vxor_vv_h, OP_SSS_H, H2, H2, H2, DO_XOR)
1358	RVVCALL(OPIVV2, vxor_vv_w, OP_SSS_W, H4, H4, H4, DO_XOR)
1359	RVVCALL(OPIVV2, vxor_vv_d, OP_SSS_D, H8, H8, H8, DO_XOR)
1360	GEN_VEXT_VV(vand_vv_b, 1)
1361	GEN_VEXT_VV(vand_vv_h, 2)
1362	GEN_VEXT_VV(vand_vv_w, 4)
1363	GEN_VEXT_VV(vand_vv_d, 8)
1364	GEN_VEXT_VV(vor_vv_b, 1)
1365	GEN_VEXT_VV(vor_vv_h, 2)
1366	GEN_VEXT_VV(vor_vv_w, 4)
1367	GEN_VEXT_VV(vor_vv_d, 8)
1368	GEN_VEXT_VV(vxor_vv_b, 1)
1369	GEN_VEXT_VV(vxor_vv_h, 2)
1370	GEN_VEXT_VV(vxor_vv_w, 4)
1371	GEN_VEXT_VV(vxor_vv_d, 8)
1372
1373	RVVCALL(OPIVX2, vand_vx_b, OP_SSS_B, H1, H1, DO_AND)
1374	RVVCALL(OPIVX2, vand_vx_h, OP_SSS_H, H2, H2, DO_AND)
1375	RVVCALL(OPIVX2, vand_vx_w, OP_SSS_W, H4, H4, DO_AND)
1376	RVVCALL(OPIVX2, vand_vx_d, OP_SSS_D, H8, H8, DO_AND)
1377	RVVCALL(OPIVX2, vor_vx_b, OP_SSS_B, H1, H1, DO_OR)
1378	RVVCALL(OPIVX2, vor_vx_h, OP_SSS_H, H2, H2, DO_OR)
1379	RVVCALL(OPIVX2, vor_vx_w, OP_SSS_W, H4, H4, DO_OR)
1380	RVVCALL(OPIVX2, vor_vx_d, OP_SSS_D, H8, H8, DO_OR)
1381	RVVCALL(OPIVX2, vxor_vx_b, OP_SSS_B, H1, H1, DO_XOR)
1382	RVVCALL(OPIVX2, vxor_vx_h, OP_SSS_H, H2, H2, DO_XOR)
1383	RVVCALL(OPIVX2, vxor_vx_w, OP_SSS_W, H4, H4, DO_XOR)
1384	RVVCALL(OPIVX2, vxor_vx_d, OP_SSS_D, H8, H8, DO_XOR)
1385	GEN_VEXT_VX(vand_vx_b, 1)
1386	GEN_VEXT_VX(vand_vx_h, 2)
1387	GEN_VEXT_VX(vand_vx_w, 4)
1388	GEN_VEXT_VX(vand_vx_d, 8)
1389	GEN_VEXT_VX(vor_vx_b, 1)
1390	GEN_VEXT_VX(vor_vx_h, 2)
1391	GEN_VEXT_VX(vor_vx_w, 4)
1392	GEN_VEXT_VX(vor_vx_d, 8)
1393	GEN_VEXT_VX(vxor_vx_b, 1)
1394	GEN_VEXT_VX(vxor_vx_h, 2)
1395	GEN_VEXT_VX(vxor_vx_w, 4)
1396	GEN_VEXT_VX(vxor_vx_d, 8)
1397
1398	/* Vector Single-Width Bit Shift Instructions */
1399	#define DO_SLL(N, M) (N << (M))
1400	#define DO_SRL(N, M) (N >> (M))
1401
1402	/* generate the helpers for shift instructions with two vector operators */
1403	#define GEN_VEXT_SHIFT_VV(NAME, TS1, TS2, HS1, HS2, OP, MASK) \
1404	void HELPER(NAME)(void vd, void v0, void *vs1, \
1405	void vs2, CPURISCVState env, uint32_t desc) \
1406	{ \
1407	uint32_t vm = vext_vm(desc); \
1408	uint32_t vl = env->vl; \
1409	uint32_t esz = sizeof(TS1); \
1410	uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
1411	uint32_t vta = vext_vta(desc); \
1412	uint32_t vma = vext_vma(desc); \
1413	uint32_t i; \
1414	\
1415	VSTART_CHECK_EARLY_EXIT(env, vl); \
1416	\
1417	for (i = env->vstart; i < vl; i++) { \
1418	if (!vm && !vext_elem_mask(v0, i)) { \
1419	/* set masked-off elements to 1s */ \
1420	vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz); \
1421	continue; \
1422	} \
1423	TS1 s1 = ((TS1 )vs1 + HS1(i)); \
1424	TS2 s2 = ((TS2 )vs2 + HS2(i)); \
1425	((TS1 )vd + HS1(i)) = OP(s2, s1 & MASK); \
1426	} \
1427	env->vstart = 0; \
1428	/* set tail elements to 1s */ \
1429	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
1430	}
1431
1432	GEN_VEXT_SHIFT_VV(vsll_vv_b, uint8_t, uint8_t, H1, H1, DO_SLL, 0x7)
1433	GEN_VEXT_SHIFT_VV(vsll_vv_h, uint16_t, uint16_t, H2, H2, DO_SLL, 0xf)
1434	GEN_VEXT_SHIFT_VV(vsll_vv_w, uint32_t, uint32_t, H4, H4, DO_SLL, 0x1f)
1435	GEN_VEXT_SHIFT_VV(vsll_vv_d, uint64_t, uint64_t, H8, H8, DO_SLL, 0x3f)
1436
1437	GEN_VEXT_SHIFT_VV(vsrl_vv_b, uint8_t, uint8_t, H1, H1, DO_SRL, 0x7)
1438	GEN_VEXT_SHIFT_VV(vsrl_vv_h, uint16_t, uint16_t, H2, H2, DO_SRL, 0xf)
1439	GEN_VEXT_SHIFT_VV(vsrl_vv_w, uint32_t, uint32_t, H4, H4, DO_SRL, 0x1f)
1440	GEN_VEXT_SHIFT_VV(vsrl_vv_d, uint64_t, uint64_t, H8, H8, DO_SRL, 0x3f)
1441
1442	GEN_VEXT_SHIFT_VV(vsra_vv_b, uint8_t, int8_t, H1, H1, DO_SRL, 0x7)
1443	GEN_VEXT_SHIFT_VV(vsra_vv_h, uint16_t, int16_t, H2, H2, DO_SRL, 0xf)
1444	GEN_VEXT_SHIFT_VV(vsra_vv_w, uint32_t, int32_t, H4, H4, DO_SRL, 0x1f)
1445	GEN_VEXT_SHIFT_VV(vsra_vv_d, uint64_t, int64_t, H8, H8, DO_SRL, 0x3f)
1446
1447	/*
1448	* generate the helpers for shift instructions with one vector and one scalar
1449	*/
1450	#define GEN_VEXT_SHIFT_VX(NAME, TD, TS2, HD, HS2, OP, MASK) \
1451	void HELPER(NAME)(void vd, void v0, target_ulong s1, \
1452	void vs2, CPURISCVState env, \
1453	uint32_t desc) \
1454	{ \
1455	uint32_t vm = vext_vm(desc); \
1456	uint32_t vl = env->vl; \
1457	uint32_t esz = sizeof(TD); \
1458	uint32_t total_elems = \
1459	vext_get_total_elems(env, desc, esz); \
1460	uint32_t vta = vext_vta(desc); \
1461	uint32_t vma = vext_vma(desc); \
1462	uint32_t i; \
1463	\
1464	VSTART_CHECK_EARLY_EXIT(env, vl); \
1465	\
1466	for (i = env->vstart; i < vl; i++) { \
1467	if (!vm && !vext_elem_mask(v0, i)) { \
1468	/* set masked-off elements to 1s */ \
1469	vext_set_elems_1s(vd, vma, i * esz, \
1470	(i + 1) * esz); \
1471	continue; \
1472	} \
1473	TS2 s2 = ((TS2 )vs2 + HS2(i)); \
1474	((TD )vd + HD(i)) = OP(s2, s1 & MASK); \
1475	} \
1476	env->vstart = 0; \
1477	/* set tail elements to 1s */ \
1478	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);\
1479	}
1480
1481	GEN_VEXT_SHIFT_VX(vsll_vx_b, uint8_t, int8_t, H1, H1, DO_SLL, 0x7)
1482	GEN_VEXT_SHIFT_VX(vsll_vx_h, uint16_t, int16_t, H2, H2, DO_SLL, 0xf)
1483	GEN_VEXT_SHIFT_VX(vsll_vx_w, uint32_t, int32_t, H4, H4, DO_SLL, 0x1f)
1484	GEN_VEXT_SHIFT_VX(vsll_vx_d, uint64_t, int64_t, H8, H8, DO_SLL, 0x3f)
1485
1486	GEN_VEXT_SHIFT_VX(vsrl_vx_b, uint8_t, uint8_t, H1, H1, DO_SRL, 0x7)
1487	GEN_VEXT_SHIFT_VX(vsrl_vx_h, uint16_t, uint16_t, H2, H2, DO_SRL, 0xf)
1488	GEN_VEXT_SHIFT_VX(vsrl_vx_w, uint32_t, uint32_t, H4, H4, DO_SRL, 0x1f)
1489	GEN_VEXT_SHIFT_VX(vsrl_vx_d, uint64_t, uint64_t, H8, H8, DO_SRL, 0x3f)
1490
1491	GEN_VEXT_SHIFT_VX(vsra_vx_b, int8_t, int8_t, H1, H1, DO_SRL, 0x7)
1492	GEN_VEXT_SHIFT_VX(vsra_vx_h, int16_t, int16_t, H2, H2, DO_SRL, 0xf)
1493	GEN_VEXT_SHIFT_VX(vsra_vx_w, int32_t, int32_t, H4, H4, DO_SRL, 0x1f)
1494	GEN_VEXT_SHIFT_VX(vsra_vx_d, int64_t, int64_t, H8, H8, DO_SRL, 0x3f)
1495
1496	/* Vector Narrowing Integer Right Shift Instructions */
1497	GEN_VEXT_SHIFT_VV(vnsrl_wv_b, uint8_t, uint16_t, H1, H2, DO_SRL, 0xf)
1498	GEN_VEXT_SHIFT_VV(vnsrl_wv_h, uint16_t, uint32_t, H2, H4, DO_SRL, 0x1f)
1499	GEN_VEXT_SHIFT_VV(vnsrl_wv_w, uint32_t, uint64_t, H4, H8, DO_SRL, 0x3f)
1500	GEN_VEXT_SHIFT_VV(vnsra_wv_b, uint8_t, int16_t, H1, H2, DO_SRL, 0xf)
1501	GEN_VEXT_SHIFT_VV(vnsra_wv_h, uint16_t, int32_t, H2, H4, DO_SRL, 0x1f)
1502	GEN_VEXT_SHIFT_VV(vnsra_wv_w, uint32_t, int64_t, H4, H8, DO_SRL, 0x3f)
1503	GEN_VEXT_SHIFT_VX(vnsrl_wx_b, uint8_t, uint16_t, H1, H2, DO_SRL, 0xf)
1504	GEN_VEXT_SHIFT_VX(vnsrl_wx_h, uint16_t, uint32_t, H2, H4, DO_SRL, 0x1f)
1505	GEN_VEXT_SHIFT_VX(vnsrl_wx_w, uint32_t, uint64_t, H4, H8, DO_SRL, 0x3f)
1506	GEN_VEXT_SHIFT_VX(vnsra_wx_b, int8_t, int16_t, H1, H2, DO_SRL, 0xf)
1507	GEN_VEXT_SHIFT_VX(vnsra_wx_h, int16_t, int32_t, H2, H4, DO_SRL, 0x1f)
1508	GEN_VEXT_SHIFT_VX(vnsra_wx_w, int32_t, int64_t, H4, H8, DO_SRL, 0x3f)
1509
1510	/* Vector Integer Comparison Instructions */
1511	#define DO_MSEQ(N, M) (N == M)
1512	#define DO_MSNE(N, M) (N != M)
1513	#define DO_MSLT(N, M) (N < M)
1514	#define DO_MSLE(N, M) (N <= M)
1515	#define DO_MSGT(N, M) (N > M)
1516
1517	#define GEN_VEXT_CMP_VV(NAME, ETYPE, H, DO_OP) \
1518	void HELPER(NAME)(void vd, void v0, void vs1, void vs2, \
1519	CPURISCVState *env, uint32_t desc) \
1520	{ \
1521	uint32_t vm = vext_vm(desc); \
1522	uint32_t vl = env->vl; \
1523	uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \
1524	uint32_t vta_all_1s = vext_vta_all_1s(desc); \
1525	uint32_t vma = vext_vma(desc); \
1526	uint32_t i; \
1527	\
1528	VSTART_CHECK_EARLY_EXIT(env, vl); \
1529	\
1530	for (i = env->vstart; i < vl; i++) { \
1531	ETYPE s1 = ((ETYPE )vs1 + H(i)); \
1532	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
1533	if (!vm && !vext_elem_mask(v0, i)) { \
1534	/* set masked-off elements to 1s */ \
1535	if (vma) { \
1536	vext_set_elem_mask(vd, i, 1); \
1537	} \
1538	continue; \
1539	} \
1540	vext_set_elem_mask(vd, i, DO_OP(s2, s1)); \
1541	} \
1542	env->vstart = 0; \
1543	/*
1544	* mask destination register are always tail-agnostic
1545	* set tail elements to 1s
1546	*/ \
1547	if (vta_all_1s) { \
1548	for (; i < total_elems; i++) { \
1549	vext_set_elem_mask(vd, i, 1); \
1550	} \
1551	} \
1552	}
1553
1554	GEN_VEXT_CMP_VV(vmseq_vv_b, uint8_t, H1, DO_MSEQ)
1555	GEN_VEXT_CMP_VV(vmseq_vv_h, uint16_t, H2, DO_MSEQ)
1556	GEN_VEXT_CMP_VV(vmseq_vv_w, uint32_t, H4, DO_MSEQ)
1557	GEN_VEXT_CMP_VV(vmseq_vv_d, uint64_t, H8, DO_MSEQ)
1558
1559	GEN_VEXT_CMP_VV(vmsne_vv_b, uint8_t, H1, DO_MSNE)
1560	GEN_VEXT_CMP_VV(vmsne_vv_h, uint16_t, H2, DO_MSNE)
1561	GEN_VEXT_CMP_VV(vmsne_vv_w, uint32_t, H4, DO_MSNE)
1562	GEN_VEXT_CMP_VV(vmsne_vv_d, uint64_t, H8, DO_MSNE)
1563
1564	GEN_VEXT_CMP_VV(vmsltu_vv_b, uint8_t, H1, DO_MSLT)
1565	GEN_VEXT_CMP_VV(vmsltu_vv_h, uint16_t, H2, DO_MSLT)
1566	GEN_VEXT_CMP_VV(vmsltu_vv_w, uint32_t, H4, DO_MSLT)
1567	GEN_VEXT_CMP_VV(vmsltu_vv_d, uint64_t, H8, DO_MSLT)
1568
1569	GEN_VEXT_CMP_VV(vmslt_vv_b, int8_t, H1, DO_MSLT)
1570	GEN_VEXT_CMP_VV(vmslt_vv_h, int16_t, H2, DO_MSLT)
1571	GEN_VEXT_CMP_VV(vmslt_vv_w, int32_t, H4, DO_MSLT)
1572	GEN_VEXT_CMP_VV(vmslt_vv_d, int64_t, H8, DO_MSLT)
1573
1574	GEN_VEXT_CMP_VV(vmsleu_vv_b, uint8_t, H1, DO_MSLE)
1575	GEN_VEXT_CMP_VV(vmsleu_vv_h, uint16_t, H2, DO_MSLE)
1576	GEN_VEXT_CMP_VV(vmsleu_vv_w, uint32_t, H4, DO_MSLE)
1577	GEN_VEXT_CMP_VV(vmsleu_vv_d, uint64_t, H8, DO_MSLE)
1578
1579	GEN_VEXT_CMP_VV(vmsle_vv_b, int8_t, H1, DO_MSLE)
1580	GEN_VEXT_CMP_VV(vmsle_vv_h, int16_t, H2, DO_MSLE)
1581	GEN_VEXT_CMP_VV(vmsle_vv_w, int32_t, H4, DO_MSLE)
1582	GEN_VEXT_CMP_VV(vmsle_vv_d, int64_t, H8, DO_MSLE)
1583
1584	#define GEN_VEXT_CMP_VX(NAME, ETYPE, H, DO_OP) \
1585	void HELPER(NAME)(void vd, void v0, target_ulong s1, void *vs2, \
1586	CPURISCVState *env, uint32_t desc) \
1587	{ \
1588	uint32_t vm = vext_vm(desc); \
1589	uint32_t vl = env->vl; \
1590	uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \
1591	uint32_t vta_all_1s = vext_vta_all_1s(desc); \
1592	uint32_t vma = vext_vma(desc); \
1593	uint32_t i; \
1594	\
1595	VSTART_CHECK_EARLY_EXIT(env, vl); \
1596	\
1597	for (i = env->vstart; i < vl; i++) { \
1598	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
1599	if (!vm && !vext_elem_mask(v0, i)) { \
1600	/* set masked-off elements to 1s */ \
1601	if (vma) { \
1602	vext_set_elem_mask(vd, i, 1); \
1603	} \
1604	continue; \
1605	} \
1606	vext_set_elem_mask(vd, i, \
1607	DO_OP(s2, (ETYPE)(target_long)s1)); \
1608	} \
1609	env->vstart = 0; \
1610	/*
1611	* mask destination register are always tail-agnostic
1612	* set tail elements to 1s
1613	*/ \
1614	if (vta_all_1s) { \
1615	for (; i < total_elems; i++) { \
1616	vext_set_elem_mask(vd, i, 1); \
1617	} \
1618	} \
1619	}
1620
1621	GEN_VEXT_CMP_VX(vmseq_vx_b, uint8_t, H1, DO_MSEQ)
1622	GEN_VEXT_CMP_VX(vmseq_vx_h, uint16_t, H2, DO_MSEQ)
1623	GEN_VEXT_CMP_VX(vmseq_vx_w, uint32_t, H4, DO_MSEQ)
1624	GEN_VEXT_CMP_VX(vmseq_vx_d, uint64_t, H8, DO_MSEQ)
1625
1626	GEN_VEXT_CMP_VX(vmsne_vx_b, uint8_t, H1, DO_MSNE)
1627	GEN_VEXT_CMP_VX(vmsne_vx_h, uint16_t, H2, DO_MSNE)
1628	GEN_VEXT_CMP_VX(vmsne_vx_w, uint32_t, H4, DO_MSNE)
1629	GEN_VEXT_CMP_VX(vmsne_vx_d, uint64_t, H8, DO_MSNE)
1630
1631	GEN_VEXT_CMP_VX(vmsltu_vx_b, uint8_t, H1, DO_MSLT)
1632	GEN_VEXT_CMP_VX(vmsltu_vx_h, uint16_t, H2, DO_MSLT)
1633	GEN_VEXT_CMP_VX(vmsltu_vx_w, uint32_t, H4, DO_MSLT)
1634	GEN_VEXT_CMP_VX(vmsltu_vx_d, uint64_t, H8, DO_MSLT)
1635
1636	GEN_VEXT_CMP_VX(vmslt_vx_b, int8_t, H1, DO_MSLT)
1637	GEN_VEXT_CMP_VX(vmslt_vx_h, int16_t, H2, DO_MSLT)
1638	GEN_VEXT_CMP_VX(vmslt_vx_w, int32_t, H4, DO_MSLT)
1639	GEN_VEXT_CMP_VX(vmslt_vx_d, int64_t, H8, DO_MSLT)
1640
1641	GEN_VEXT_CMP_VX(vmsleu_vx_b, uint8_t, H1, DO_MSLE)
1642	GEN_VEXT_CMP_VX(vmsleu_vx_h, uint16_t, H2, DO_MSLE)
1643	GEN_VEXT_CMP_VX(vmsleu_vx_w, uint32_t, H4, DO_MSLE)
1644	GEN_VEXT_CMP_VX(vmsleu_vx_d, uint64_t, H8, DO_MSLE)
1645
1646	GEN_VEXT_CMP_VX(vmsle_vx_b, int8_t, H1, DO_MSLE)
1647	GEN_VEXT_CMP_VX(vmsle_vx_h, int16_t, H2, DO_MSLE)
1648	GEN_VEXT_CMP_VX(vmsle_vx_w, int32_t, H4, DO_MSLE)
1649	GEN_VEXT_CMP_VX(vmsle_vx_d, int64_t, H8, DO_MSLE)
1650
1651	GEN_VEXT_CMP_VX(vmsgtu_vx_b, uint8_t, H1, DO_MSGT)
1652	GEN_VEXT_CMP_VX(vmsgtu_vx_h, uint16_t, H2, DO_MSGT)
1653	GEN_VEXT_CMP_VX(vmsgtu_vx_w, uint32_t, H4, DO_MSGT)
1654	GEN_VEXT_CMP_VX(vmsgtu_vx_d, uint64_t, H8, DO_MSGT)
1655
1656	GEN_VEXT_CMP_VX(vmsgt_vx_b, int8_t, H1, DO_MSGT)
1657	GEN_VEXT_CMP_VX(vmsgt_vx_h, int16_t, H2, DO_MSGT)
1658	GEN_VEXT_CMP_VX(vmsgt_vx_w, int32_t, H4, DO_MSGT)
1659	GEN_VEXT_CMP_VX(vmsgt_vx_d, int64_t, H8, DO_MSGT)
1660
1661	/* Vector Integer Min/Max Instructions */
1662	RVVCALL(OPIVV2, vminu_vv_b, OP_UUU_B, H1, H1, H1, DO_MIN)
1663	RVVCALL(OPIVV2, vminu_vv_h, OP_UUU_H, H2, H2, H2, DO_MIN)
1664	RVVCALL(OPIVV2, vminu_vv_w, OP_UUU_W, H4, H4, H4, DO_MIN)
1665	RVVCALL(OPIVV2, vminu_vv_d, OP_UUU_D, H8, H8, H8, DO_MIN)
1666	RVVCALL(OPIVV2, vmin_vv_b, OP_SSS_B, H1, H1, H1, DO_MIN)
1667	RVVCALL(OPIVV2, vmin_vv_h, OP_SSS_H, H2, H2, H2, DO_MIN)
1668	RVVCALL(OPIVV2, vmin_vv_w, OP_SSS_W, H4, H4, H4, DO_MIN)
1669	RVVCALL(OPIVV2, vmin_vv_d, OP_SSS_D, H8, H8, H8, DO_MIN)
1670	RVVCALL(OPIVV2, vmaxu_vv_b, OP_UUU_B, H1, H1, H1, DO_MAX)
1671	RVVCALL(OPIVV2, vmaxu_vv_h, OP_UUU_H, H2, H2, H2, DO_MAX)
1672	RVVCALL(OPIVV2, vmaxu_vv_w, OP_UUU_W, H4, H4, H4, DO_MAX)
1673	RVVCALL(OPIVV2, vmaxu_vv_d, OP_UUU_D, H8, H8, H8, DO_MAX)
1674	RVVCALL(OPIVV2, vmax_vv_b, OP_SSS_B, H1, H1, H1, DO_MAX)
1675	RVVCALL(OPIVV2, vmax_vv_h, OP_SSS_H, H2, H2, H2, DO_MAX)
1676	RVVCALL(OPIVV2, vmax_vv_w, OP_SSS_W, H4, H4, H4, DO_MAX)
1677	RVVCALL(OPIVV2, vmax_vv_d, OP_SSS_D, H8, H8, H8, DO_MAX)
1678	GEN_VEXT_VV(vminu_vv_b, 1)
1679	GEN_VEXT_VV(vminu_vv_h, 2)
1680	GEN_VEXT_VV(vminu_vv_w, 4)
1681	GEN_VEXT_VV(vminu_vv_d, 8)
1682	GEN_VEXT_VV(vmin_vv_b, 1)
1683	GEN_VEXT_VV(vmin_vv_h, 2)
1684	GEN_VEXT_VV(vmin_vv_w, 4)
1685	GEN_VEXT_VV(vmin_vv_d, 8)
1686	GEN_VEXT_VV(vmaxu_vv_b, 1)
1687	GEN_VEXT_VV(vmaxu_vv_h, 2)
1688	GEN_VEXT_VV(vmaxu_vv_w, 4)
1689	GEN_VEXT_VV(vmaxu_vv_d, 8)
1690	GEN_VEXT_VV(vmax_vv_b, 1)
1691	GEN_VEXT_VV(vmax_vv_h, 2)
1692	GEN_VEXT_VV(vmax_vv_w, 4)
1693	GEN_VEXT_VV(vmax_vv_d, 8)
1694
1695	RVVCALL(OPIVX2, vminu_vx_b, OP_UUU_B, H1, H1, DO_MIN)
1696	RVVCALL(OPIVX2, vminu_vx_h, OP_UUU_H, H2, H2, DO_MIN)
1697	RVVCALL(OPIVX2, vminu_vx_w, OP_UUU_W, H4, H4, DO_MIN)
1698	RVVCALL(OPIVX2, vminu_vx_d, OP_UUU_D, H8, H8, DO_MIN)
1699	RVVCALL(OPIVX2, vmin_vx_b, OP_SSS_B, H1, H1, DO_MIN)
1700	RVVCALL(OPIVX2, vmin_vx_h, OP_SSS_H, H2, H2, DO_MIN)
1701	RVVCALL(OPIVX2, vmin_vx_w, OP_SSS_W, H4, H4, DO_MIN)
1702	RVVCALL(OPIVX2, vmin_vx_d, OP_SSS_D, H8, H8, DO_MIN)
1703	RVVCALL(OPIVX2, vmaxu_vx_b, OP_UUU_B, H1, H1, DO_MAX)
1704	RVVCALL(OPIVX2, vmaxu_vx_h, OP_UUU_H, H2, H2, DO_MAX)
1705	RVVCALL(OPIVX2, vmaxu_vx_w, OP_UUU_W, H4, H4, DO_MAX)
1706	RVVCALL(OPIVX2, vmaxu_vx_d, OP_UUU_D, H8, H8, DO_MAX)
1707	RVVCALL(OPIVX2, vmax_vx_b, OP_SSS_B, H1, H1, DO_MAX)
1708	RVVCALL(OPIVX2, vmax_vx_h, OP_SSS_H, H2, H2, DO_MAX)
1709	RVVCALL(OPIVX2, vmax_vx_w, OP_SSS_W, H4, H4, DO_MAX)
1710	RVVCALL(OPIVX2, vmax_vx_d, OP_SSS_D, H8, H8, DO_MAX)
1711	GEN_VEXT_VX(vminu_vx_b, 1)
1712	GEN_VEXT_VX(vminu_vx_h, 2)
1713	GEN_VEXT_VX(vminu_vx_w, 4)
1714	GEN_VEXT_VX(vminu_vx_d, 8)
1715	GEN_VEXT_VX(vmin_vx_b, 1)
1716	GEN_VEXT_VX(vmin_vx_h, 2)
1717	GEN_VEXT_VX(vmin_vx_w, 4)
1718	GEN_VEXT_VX(vmin_vx_d, 8)
1719	GEN_VEXT_VX(vmaxu_vx_b, 1)
1720	GEN_VEXT_VX(vmaxu_vx_h, 2)
1721	GEN_VEXT_VX(vmaxu_vx_w, 4)
1722	GEN_VEXT_VX(vmaxu_vx_d, 8)
1723	GEN_VEXT_VX(vmax_vx_b, 1)
1724	GEN_VEXT_VX(vmax_vx_h, 2)
1725	GEN_VEXT_VX(vmax_vx_w, 4)
1726	GEN_VEXT_VX(vmax_vx_d, 8)
1727
1728	/* Vector Single-Width Integer Multiply Instructions */
1729	#define DO_MUL(N, M) (N * M)
1730	RVVCALL(OPIVV2, vmul_vv_b, OP_SSS_B, H1, H1, H1, DO_MUL)
1731	RVVCALL(OPIVV2, vmul_vv_h, OP_SSS_H, H2, H2, H2, DO_MUL)
1732	RVVCALL(OPIVV2, vmul_vv_w, OP_SSS_W, H4, H4, H4, DO_MUL)
1733	RVVCALL(OPIVV2, vmul_vv_d, OP_SSS_D, H8, H8, H8, DO_MUL)
1734	GEN_VEXT_VV(vmul_vv_b, 1)
1735	GEN_VEXT_VV(vmul_vv_h, 2)
1736	GEN_VEXT_VV(vmul_vv_w, 4)
1737	GEN_VEXT_VV(vmul_vv_d, 8)
1738
1739	static int8_t do_mulh_b(int8_t s2, int8_t s1)
1740	{
1741	return (int16_t)s2 * (int16_t)s1 >> 8;
1742	}
1743
1744	static int16_t do_mulh_h(int16_t s2, int16_t s1)
1745	{
1746	return (int32_t)s2 * (int32_t)s1 >> 16;
1747	}
1748
1749	static int32_t do_mulh_w(int32_t s2, int32_t s1)
1750	{
1751	return (int64_t)s2 * (int64_t)s1 >> 32;
1752	}
1753
1754	static int64_t do_mulh_d(int64_t s2, int64_t s1)
1755	{
1756	uint64_t hi_64, lo_64;
1757
1758	muls64(&lo_64, &hi_64, s1, s2);
1759	return hi_64;
1760	}
1761
1762	static uint8_t do_mulhu_b(uint8_t s2, uint8_t s1)
1763	{
1764	return (uint16_t)s2 * (uint16_t)s1 >> 8;
1765	}
1766
1767	static uint16_t do_mulhu_h(uint16_t s2, uint16_t s1)
1768	{
1769	return (uint32_t)s2 * (uint32_t)s1 >> 16;
1770	}
1771
1772	static uint32_t do_mulhu_w(uint32_t s2, uint32_t s1)
1773	{
1774	return (uint64_t)s2 * (uint64_t)s1 >> 32;
1775	}
1776
1777	static uint64_t do_mulhu_d(uint64_t s2, uint64_t s1)
1778	{
1779	uint64_t hi_64, lo_64;
1780
1781	mulu64(&lo_64, &hi_64, s2, s1);
1782	return hi_64;
1783	}
1784
1785	static int8_t do_mulhsu_b(int8_t s2, uint8_t s1)
1786	{
1787	return (int16_t)s2 * (uint16_t)s1 >> 8;
1788	}
1789
1790	static int16_t do_mulhsu_h(int16_t s2, uint16_t s1)
1791	{
1792	return (int32_t)s2 * (uint32_t)s1 >> 16;
1793	}
1794
1795	static int32_t do_mulhsu_w(int32_t s2, uint32_t s1)
1796	{
1797	return (int64_t)s2 * (uint64_t)s1 >> 32;
1798	}
1799
1800	/*
1801	* Let A = signed operand,
1802	* B = unsigned operand
1803	* P = mulu64(A, B), unsigned product
1804	*
1805	* LET X = 2 ** 64 - A, 2's complement of A
1806	* SP = signed product
1807	* THEN
1808	* IF A < 0
1809	* SP = -X * B
1810	* = -(2 ** 64 - A) * B
1811	* = A * B - 2 ** 64 * B
1812	* = P - 2 ** 64 * B
1813	* ELSE
1814	* SP = P
1815	* THEN
1816	* HI_P -= (A < 0 ? B : 0)
1817	*/
1818
1819	static int64_t do_mulhsu_d(int64_t s2, uint64_t s1)
1820	{
1821	uint64_t hi_64, lo_64;
1822
1823	mulu64(&lo_64, &hi_64, s2, s1);
1824
1825	hi_64 -= s2 < 0 ? s1 : 0;
1826	return hi_64;
1827	}
1828
1829	RVVCALL(OPIVV2, vmulh_vv_b, OP_SSS_B, H1, H1, H1, do_mulh_b)
1830	RVVCALL(OPIVV2, vmulh_vv_h, OP_SSS_H, H2, H2, H2, do_mulh_h)
1831	RVVCALL(OPIVV2, vmulh_vv_w, OP_SSS_W, H4, H4, H4, do_mulh_w)
1832	RVVCALL(OPIVV2, vmulh_vv_d, OP_SSS_D, H8, H8, H8, do_mulh_d)
1833	RVVCALL(OPIVV2, vmulhu_vv_b, OP_UUU_B, H1, H1, H1, do_mulhu_b)
1834	RVVCALL(OPIVV2, vmulhu_vv_h, OP_UUU_H, H2, H2, H2, do_mulhu_h)
1835	RVVCALL(OPIVV2, vmulhu_vv_w, OP_UUU_W, H4, H4, H4, do_mulhu_w)
1836	RVVCALL(OPIVV2, vmulhu_vv_d, OP_UUU_D, H8, H8, H8, do_mulhu_d)
1837	RVVCALL(OPIVV2, vmulhsu_vv_b, OP_SUS_B, H1, H1, H1, do_mulhsu_b)
1838	RVVCALL(OPIVV2, vmulhsu_vv_h, OP_SUS_H, H2, H2, H2, do_mulhsu_h)
1839	RVVCALL(OPIVV2, vmulhsu_vv_w, OP_SUS_W, H4, H4, H4, do_mulhsu_w)
1840	RVVCALL(OPIVV2, vmulhsu_vv_d, OP_SUS_D, H8, H8, H8, do_mulhsu_d)
1841	GEN_VEXT_VV(vmulh_vv_b, 1)
1842	GEN_VEXT_VV(vmulh_vv_h, 2)
1843	GEN_VEXT_VV(vmulh_vv_w, 4)
1844	GEN_VEXT_VV(vmulh_vv_d, 8)
1845	GEN_VEXT_VV(vmulhu_vv_b, 1)
1846	GEN_VEXT_VV(vmulhu_vv_h, 2)
1847	GEN_VEXT_VV(vmulhu_vv_w, 4)
1848	GEN_VEXT_VV(vmulhu_vv_d, 8)
1849	GEN_VEXT_VV(vmulhsu_vv_b, 1)
1850	GEN_VEXT_VV(vmulhsu_vv_h, 2)
1851	GEN_VEXT_VV(vmulhsu_vv_w, 4)
1852	GEN_VEXT_VV(vmulhsu_vv_d, 8)
1853
1854	RVVCALL(OPIVX2, vmul_vx_b, OP_SSS_B, H1, H1, DO_MUL)
1855	RVVCALL(OPIVX2, vmul_vx_h, OP_SSS_H, H2, H2, DO_MUL)
1856	RVVCALL(OPIVX2, vmul_vx_w, OP_SSS_W, H4, H4, DO_MUL)
1857	RVVCALL(OPIVX2, vmul_vx_d, OP_SSS_D, H8, H8, DO_MUL)
1858	RVVCALL(OPIVX2, vmulh_vx_b, OP_SSS_B, H1, H1, do_mulh_b)
1859	RVVCALL(OPIVX2, vmulh_vx_h, OP_SSS_H, H2, H2, do_mulh_h)
1860	RVVCALL(OPIVX2, vmulh_vx_w, OP_SSS_W, H4, H4, do_mulh_w)
1861	RVVCALL(OPIVX2, vmulh_vx_d, OP_SSS_D, H8, H8, do_mulh_d)
1862	RVVCALL(OPIVX2, vmulhu_vx_b, OP_UUU_B, H1, H1, do_mulhu_b)
1863	RVVCALL(OPIVX2, vmulhu_vx_h, OP_UUU_H, H2, H2, do_mulhu_h)
1864	RVVCALL(OPIVX2, vmulhu_vx_w, OP_UUU_W, H4, H4, do_mulhu_w)
1865	RVVCALL(OPIVX2, vmulhu_vx_d, OP_UUU_D, H8, H8, do_mulhu_d)
1866	RVVCALL(OPIVX2, vmulhsu_vx_b, OP_SUS_B, H1, H1, do_mulhsu_b)
1867	RVVCALL(OPIVX2, vmulhsu_vx_h, OP_SUS_H, H2, H2, do_mulhsu_h)
1868	RVVCALL(OPIVX2, vmulhsu_vx_w, OP_SUS_W, H4, H4, do_mulhsu_w)
1869	RVVCALL(OPIVX2, vmulhsu_vx_d, OP_SUS_D, H8, H8, do_mulhsu_d)
1870	GEN_VEXT_VX(vmul_vx_b, 1)
1871	GEN_VEXT_VX(vmul_vx_h, 2)
1872	GEN_VEXT_VX(vmul_vx_w, 4)
1873	GEN_VEXT_VX(vmul_vx_d, 8)
1874	GEN_VEXT_VX(vmulh_vx_b, 1)
1875	GEN_VEXT_VX(vmulh_vx_h, 2)
1876	GEN_VEXT_VX(vmulh_vx_w, 4)
1877	GEN_VEXT_VX(vmulh_vx_d, 8)
1878	GEN_VEXT_VX(vmulhu_vx_b, 1)
1879	GEN_VEXT_VX(vmulhu_vx_h, 2)
1880	GEN_VEXT_VX(vmulhu_vx_w, 4)
1881	GEN_VEXT_VX(vmulhu_vx_d, 8)
1882	GEN_VEXT_VX(vmulhsu_vx_b, 1)
1883	GEN_VEXT_VX(vmulhsu_vx_h, 2)
1884	GEN_VEXT_VX(vmulhsu_vx_w, 4)
1885	GEN_VEXT_VX(vmulhsu_vx_d, 8)
1886
1887	/* Vector Integer Divide Instructions */
1888	#define DO_DIVU(N, M) (unlikely(M == 0) ? (__typeof(N))(-1) : N / M)
1889	#define DO_REMU(N, M) (unlikely(M == 0) ? N : N % M)
1890	#define DO_DIV(N, M) (unlikely(M == 0) ? (__typeof(N))(-1) : \
1891	unlikely((N == -N) && (M == (__typeof(N))(-1))) ? N : N / M)
1892	#define DO_REM(N, M) (unlikely(M == 0) ? N : \
1893	unlikely((N == -N) && (M == (__typeof(N))(-1))) ? 0 : N % M)
1894
1895	RVVCALL(OPIVV2, vdivu_vv_b, OP_UUU_B, H1, H1, H1, DO_DIVU)
1896	RVVCALL(OPIVV2, vdivu_vv_h, OP_UUU_H, H2, H2, H2, DO_DIVU)
1897	RVVCALL(OPIVV2, vdivu_vv_w, OP_UUU_W, H4, H4, H4, DO_DIVU)
1898	RVVCALL(OPIVV2, vdivu_vv_d, OP_UUU_D, H8, H8, H8, DO_DIVU)
1899	RVVCALL(OPIVV2, vdiv_vv_b, OP_SSS_B, H1, H1, H1, DO_DIV)
1900	RVVCALL(OPIVV2, vdiv_vv_h, OP_SSS_H, H2, H2, H2, DO_DIV)
1901	RVVCALL(OPIVV2, vdiv_vv_w, OP_SSS_W, H4, H4, H4, DO_DIV)
1902	RVVCALL(OPIVV2, vdiv_vv_d, OP_SSS_D, H8, H8, H8, DO_DIV)
1903	RVVCALL(OPIVV2, vremu_vv_b, OP_UUU_B, H1, H1, H1, DO_REMU)
1904	RVVCALL(OPIVV2, vremu_vv_h, OP_UUU_H, H2, H2, H2, DO_REMU)
1905	RVVCALL(OPIVV2, vremu_vv_w, OP_UUU_W, H4, H4, H4, DO_REMU)
1906	RVVCALL(OPIVV2, vremu_vv_d, OP_UUU_D, H8, H8, H8, DO_REMU)
1907	RVVCALL(OPIVV2, vrem_vv_b, OP_SSS_B, H1, H1, H1, DO_REM)
1908	RVVCALL(OPIVV2, vrem_vv_h, OP_SSS_H, H2, H2, H2, DO_REM)
1909	RVVCALL(OPIVV2, vrem_vv_w, OP_SSS_W, H4, H4, H4, DO_REM)
1910	RVVCALL(OPIVV2, vrem_vv_d, OP_SSS_D, H8, H8, H8, DO_REM)
1911	GEN_VEXT_VV(vdivu_vv_b, 1)
1912	GEN_VEXT_VV(vdivu_vv_h, 2)
1913	GEN_VEXT_VV(vdivu_vv_w, 4)
1914	GEN_VEXT_VV(vdivu_vv_d, 8)
1915	GEN_VEXT_VV(vdiv_vv_b, 1)
1916	GEN_VEXT_VV(vdiv_vv_h, 2)
1917	GEN_VEXT_VV(vdiv_vv_w, 4)
1918	GEN_VEXT_VV(vdiv_vv_d, 8)
1919	GEN_VEXT_VV(vremu_vv_b, 1)
1920	GEN_VEXT_VV(vremu_vv_h, 2)
1921	GEN_VEXT_VV(vremu_vv_w, 4)
1922	GEN_VEXT_VV(vremu_vv_d, 8)
1923	GEN_VEXT_VV(vrem_vv_b, 1)
1924	GEN_VEXT_VV(vrem_vv_h, 2)
1925	GEN_VEXT_VV(vrem_vv_w, 4)
1926	GEN_VEXT_VV(vrem_vv_d, 8)
1927
1928	RVVCALL(OPIVX2, vdivu_vx_b, OP_UUU_B, H1, H1, DO_DIVU)
1929	RVVCALL(OPIVX2, vdivu_vx_h, OP_UUU_H, H2, H2, DO_DIVU)
1930	RVVCALL(OPIVX2, vdivu_vx_w, OP_UUU_W, H4, H4, DO_DIVU)
1931	RVVCALL(OPIVX2, vdivu_vx_d, OP_UUU_D, H8, H8, DO_DIVU)
1932	RVVCALL(OPIVX2, vdiv_vx_b, OP_SSS_B, H1, H1, DO_DIV)
1933	RVVCALL(OPIVX2, vdiv_vx_h, OP_SSS_H, H2, H2, DO_DIV)
1934	RVVCALL(OPIVX2, vdiv_vx_w, OP_SSS_W, H4, H4, DO_DIV)
1935	RVVCALL(OPIVX2, vdiv_vx_d, OP_SSS_D, H8, H8, DO_DIV)
1936	RVVCALL(OPIVX2, vremu_vx_b, OP_UUU_B, H1, H1, DO_REMU)
1937	RVVCALL(OPIVX2, vremu_vx_h, OP_UUU_H, H2, H2, DO_REMU)
1938	RVVCALL(OPIVX2, vremu_vx_w, OP_UUU_W, H4, H4, DO_REMU)
1939	RVVCALL(OPIVX2, vremu_vx_d, OP_UUU_D, H8, H8, DO_REMU)
1940	RVVCALL(OPIVX2, vrem_vx_b, OP_SSS_B, H1, H1, DO_REM)
1941	RVVCALL(OPIVX2, vrem_vx_h, OP_SSS_H, H2, H2, DO_REM)
1942	RVVCALL(OPIVX2, vrem_vx_w, OP_SSS_W, H4, H4, DO_REM)
1943	RVVCALL(OPIVX2, vrem_vx_d, OP_SSS_D, H8, H8, DO_REM)
1944	GEN_VEXT_VX(vdivu_vx_b, 1)
1945	GEN_VEXT_VX(vdivu_vx_h, 2)
1946	GEN_VEXT_VX(vdivu_vx_w, 4)
1947	GEN_VEXT_VX(vdivu_vx_d, 8)
1948	GEN_VEXT_VX(vdiv_vx_b, 1)
1949	GEN_VEXT_VX(vdiv_vx_h, 2)
1950	GEN_VEXT_VX(vdiv_vx_w, 4)
1951	GEN_VEXT_VX(vdiv_vx_d, 8)
1952	GEN_VEXT_VX(vremu_vx_b, 1)
1953	GEN_VEXT_VX(vremu_vx_h, 2)
1954	GEN_VEXT_VX(vremu_vx_w, 4)
1955	GEN_VEXT_VX(vremu_vx_d, 8)
1956	GEN_VEXT_VX(vrem_vx_b, 1)
1957	GEN_VEXT_VX(vrem_vx_h, 2)
1958	GEN_VEXT_VX(vrem_vx_w, 4)
1959	GEN_VEXT_VX(vrem_vx_d, 8)
1960
1961	/* Vector Widening Integer Multiply Instructions */
1962	RVVCALL(OPIVV2, vwmul_vv_b, WOP_SSS_B, H2, H1, H1, DO_MUL)
1963	RVVCALL(OPIVV2, vwmul_vv_h, WOP_SSS_H, H4, H2, H2, DO_MUL)
1964	RVVCALL(OPIVV2, vwmul_vv_w, WOP_SSS_W, H8, H4, H4, DO_MUL)
1965	RVVCALL(OPIVV2, vwmulu_vv_b, WOP_UUU_B, H2, H1, H1, DO_MUL)
1966	RVVCALL(OPIVV2, vwmulu_vv_h, WOP_UUU_H, H4, H2, H2, DO_MUL)
1967	RVVCALL(OPIVV2, vwmulu_vv_w, WOP_UUU_W, H8, H4, H4, DO_MUL)
1968	RVVCALL(OPIVV2, vwmulsu_vv_b, WOP_SUS_B, H2, H1, H1, DO_MUL)
1969	RVVCALL(OPIVV2, vwmulsu_vv_h, WOP_SUS_H, H4, H2, H2, DO_MUL)
1970	RVVCALL(OPIVV2, vwmulsu_vv_w, WOP_SUS_W, H8, H4, H4, DO_MUL)
1971	GEN_VEXT_VV(vwmul_vv_b, 2)
1972	GEN_VEXT_VV(vwmul_vv_h, 4)
1973	GEN_VEXT_VV(vwmul_vv_w, 8)
1974	GEN_VEXT_VV(vwmulu_vv_b, 2)
1975	GEN_VEXT_VV(vwmulu_vv_h, 4)
1976	GEN_VEXT_VV(vwmulu_vv_w, 8)
1977	GEN_VEXT_VV(vwmulsu_vv_b, 2)
1978	GEN_VEXT_VV(vwmulsu_vv_h, 4)
1979	GEN_VEXT_VV(vwmulsu_vv_w, 8)
1980
1981	RVVCALL(OPIVX2, vwmul_vx_b, WOP_SSS_B, H2, H1, DO_MUL)
1982	RVVCALL(OPIVX2, vwmul_vx_h, WOP_SSS_H, H4, H2, DO_MUL)
1983	RVVCALL(OPIVX2, vwmul_vx_w, WOP_SSS_W, H8, H4, DO_MUL)
1984	RVVCALL(OPIVX2, vwmulu_vx_b, WOP_UUU_B, H2, H1, DO_MUL)
1985	RVVCALL(OPIVX2, vwmulu_vx_h, WOP_UUU_H, H4, H2, DO_MUL)
1986	RVVCALL(OPIVX2, vwmulu_vx_w, WOP_UUU_W, H8, H4, DO_MUL)
1987	RVVCALL(OPIVX2, vwmulsu_vx_b, WOP_SUS_B, H2, H1, DO_MUL)
1988	RVVCALL(OPIVX2, vwmulsu_vx_h, WOP_SUS_H, H4, H2, DO_MUL)
1989	RVVCALL(OPIVX2, vwmulsu_vx_w, WOP_SUS_W, H8, H4, DO_MUL)
1990	GEN_VEXT_VX(vwmul_vx_b, 2)
1991	GEN_VEXT_VX(vwmul_vx_h, 4)
1992	GEN_VEXT_VX(vwmul_vx_w, 8)
1993	GEN_VEXT_VX(vwmulu_vx_b, 2)
1994	GEN_VEXT_VX(vwmulu_vx_h, 4)
1995	GEN_VEXT_VX(vwmulu_vx_w, 8)
1996	GEN_VEXT_VX(vwmulsu_vx_b, 2)
1997	GEN_VEXT_VX(vwmulsu_vx_h, 4)
1998	GEN_VEXT_VX(vwmulsu_vx_w, 8)
1999
2000	/* Vector Single-Width Integer Multiply-Add Instructions */
2001	#define OPIVV3(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
2002	static void do_##NAME(void vd, void vs1, void *vs2, int i) \
2003	{ \
2004	TX1 s1 = ((T1 )vs1 + HS1(i)); \
2005	TX2 s2 = ((T2 )vs2 + HS2(i)); \
2006	TD d = ((TD )vd + HD(i)); \
2007	((TD )vd + HD(i)) = OP(s2, s1, d); \
2008	}
2009
2010	#define DO_MACC(N, M, D) (M * N + D)
2011	#define DO_NMSAC(N, M, D) (-(M * N) + D)
2012	#define DO_MADD(N, M, D) (M * D + N)
2013	#define DO_NMSUB(N, M, D) (-(M * D) + N)
2014	RVVCALL(OPIVV3, vmacc_vv_b, OP_SSS_B, H1, H1, H1, DO_MACC)
2015	RVVCALL(OPIVV3, vmacc_vv_h, OP_SSS_H, H2, H2, H2, DO_MACC)
2016	RVVCALL(OPIVV3, vmacc_vv_w, OP_SSS_W, H4, H4, H4, DO_MACC)
2017	RVVCALL(OPIVV3, vmacc_vv_d, OP_SSS_D, H8, H8, H8, DO_MACC)
2018	RVVCALL(OPIVV3, vnmsac_vv_b, OP_SSS_B, H1, H1, H1, DO_NMSAC)
2019	RVVCALL(OPIVV3, vnmsac_vv_h, OP_SSS_H, H2, H2, H2, DO_NMSAC)
2020	RVVCALL(OPIVV3, vnmsac_vv_w, OP_SSS_W, H4, H4, H4, DO_NMSAC)
2021	RVVCALL(OPIVV3, vnmsac_vv_d, OP_SSS_D, H8, H8, H8, DO_NMSAC)
2022	RVVCALL(OPIVV3, vmadd_vv_b, OP_SSS_B, H1, H1, H1, DO_MADD)
2023	RVVCALL(OPIVV3, vmadd_vv_h, OP_SSS_H, H2, H2, H2, DO_MADD)
2024	RVVCALL(OPIVV3, vmadd_vv_w, OP_SSS_W, H4, H4, H4, DO_MADD)
2025	RVVCALL(OPIVV3, vmadd_vv_d, OP_SSS_D, H8, H8, H8, DO_MADD)
2026	RVVCALL(OPIVV3, vnmsub_vv_b, OP_SSS_B, H1, H1, H1, DO_NMSUB)
2027	RVVCALL(OPIVV3, vnmsub_vv_h, OP_SSS_H, H2, H2, H2, DO_NMSUB)
2028	RVVCALL(OPIVV3, vnmsub_vv_w, OP_SSS_W, H4, H4, H4, DO_NMSUB)
2029	RVVCALL(OPIVV3, vnmsub_vv_d, OP_SSS_D, H8, H8, H8, DO_NMSUB)
2030	GEN_VEXT_VV(vmacc_vv_b, 1)
2031	GEN_VEXT_VV(vmacc_vv_h, 2)
2032	GEN_VEXT_VV(vmacc_vv_w, 4)
2033	GEN_VEXT_VV(vmacc_vv_d, 8)
2034	GEN_VEXT_VV(vnmsac_vv_b, 1)
2035	GEN_VEXT_VV(vnmsac_vv_h, 2)
2036	GEN_VEXT_VV(vnmsac_vv_w, 4)
2037	GEN_VEXT_VV(vnmsac_vv_d, 8)
2038	GEN_VEXT_VV(vmadd_vv_b, 1)
2039	GEN_VEXT_VV(vmadd_vv_h, 2)
2040	GEN_VEXT_VV(vmadd_vv_w, 4)
2041	GEN_VEXT_VV(vmadd_vv_d, 8)
2042	GEN_VEXT_VV(vnmsub_vv_b, 1)
2043	GEN_VEXT_VV(vnmsub_vv_h, 2)
2044	GEN_VEXT_VV(vnmsub_vv_w, 4)
2045	GEN_VEXT_VV(vnmsub_vv_d, 8)
2046
2047	#define OPIVX3(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \
2048	static void do_##NAME(void vd, target_long s1, void vs2, int i) \
2049	{ \
2050	TX2 s2 = ((T2 )vs2 + HS2(i)); \
2051	TD d = ((TD )vd + HD(i)); \
2052	((TD )vd + HD(i)) = OP(s2, (TX1)(T1)s1, d); \
2053	}
2054
2055	RVVCALL(OPIVX3, vmacc_vx_b, OP_SSS_B, H1, H1, DO_MACC)
2056	RVVCALL(OPIVX3, vmacc_vx_h, OP_SSS_H, H2, H2, DO_MACC)
2057	RVVCALL(OPIVX3, vmacc_vx_w, OP_SSS_W, H4, H4, DO_MACC)
2058	RVVCALL(OPIVX3, vmacc_vx_d, OP_SSS_D, H8, H8, DO_MACC)
2059	RVVCALL(OPIVX3, vnmsac_vx_b, OP_SSS_B, H1, H1, DO_NMSAC)
2060	RVVCALL(OPIVX3, vnmsac_vx_h, OP_SSS_H, H2, H2, DO_NMSAC)
2061	RVVCALL(OPIVX3, vnmsac_vx_w, OP_SSS_W, H4, H4, DO_NMSAC)
2062	RVVCALL(OPIVX3, vnmsac_vx_d, OP_SSS_D, H8, H8, DO_NMSAC)
2063	RVVCALL(OPIVX3, vmadd_vx_b, OP_SSS_B, H1, H1, DO_MADD)
2064	RVVCALL(OPIVX3, vmadd_vx_h, OP_SSS_H, H2, H2, DO_MADD)
2065	RVVCALL(OPIVX3, vmadd_vx_w, OP_SSS_W, H4, H4, DO_MADD)
2066	RVVCALL(OPIVX3, vmadd_vx_d, OP_SSS_D, H8, H8, DO_MADD)
2067	RVVCALL(OPIVX3, vnmsub_vx_b, OP_SSS_B, H1, H1, DO_NMSUB)
2068	RVVCALL(OPIVX3, vnmsub_vx_h, OP_SSS_H, H2, H2, DO_NMSUB)
2069	RVVCALL(OPIVX3, vnmsub_vx_w, OP_SSS_W, H4, H4, DO_NMSUB)
2070	RVVCALL(OPIVX3, vnmsub_vx_d, OP_SSS_D, H8, H8, DO_NMSUB)
2071	GEN_VEXT_VX(vmacc_vx_b, 1)
2072	GEN_VEXT_VX(vmacc_vx_h, 2)
2073	GEN_VEXT_VX(vmacc_vx_w, 4)
2074	GEN_VEXT_VX(vmacc_vx_d, 8)
2075	GEN_VEXT_VX(vnmsac_vx_b, 1)
2076	GEN_VEXT_VX(vnmsac_vx_h, 2)
2077	GEN_VEXT_VX(vnmsac_vx_w, 4)
2078	GEN_VEXT_VX(vnmsac_vx_d, 8)
2079	GEN_VEXT_VX(vmadd_vx_b, 1)
2080	GEN_VEXT_VX(vmadd_vx_h, 2)
2081	GEN_VEXT_VX(vmadd_vx_w, 4)
2082	GEN_VEXT_VX(vmadd_vx_d, 8)
2083	GEN_VEXT_VX(vnmsub_vx_b, 1)
2084	GEN_VEXT_VX(vnmsub_vx_h, 2)
2085	GEN_VEXT_VX(vnmsub_vx_w, 4)
2086	GEN_VEXT_VX(vnmsub_vx_d, 8)
2087
2088	/* Vector Widening Integer Multiply-Add Instructions */
2089	RVVCALL(OPIVV3, vwmaccu_vv_b, WOP_UUU_B, H2, H1, H1, DO_MACC)
2090	RVVCALL(OPIVV3, vwmaccu_vv_h, WOP_UUU_H, H4, H2, H2, DO_MACC)
2091	RVVCALL(OPIVV3, vwmaccu_vv_w, WOP_UUU_W, H8, H4, H4, DO_MACC)
2092	RVVCALL(OPIVV3, vwmacc_vv_b, WOP_SSS_B, H2, H1, H1, DO_MACC)
2093	RVVCALL(OPIVV3, vwmacc_vv_h, WOP_SSS_H, H4, H2, H2, DO_MACC)
2094	RVVCALL(OPIVV3, vwmacc_vv_w, WOP_SSS_W, H8, H4, H4, DO_MACC)
2095	RVVCALL(OPIVV3, vwmaccsu_vv_b, WOP_SSU_B, H2, H1, H1, DO_MACC)
2096	RVVCALL(OPIVV3, vwmaccsu_vv_h, WOP_SSU_H, H4, H2, H2, DO_MACC)
2097	RVVCALL(OPIVV3, vwmaccsu_vv_w, WOP_SSU_W, H8, H4, H4, DO_MACC)
2098	GEN_VEXT_VV(vwmaccu_vv_b, 2)
2099	GEN_VEXT_VV(vwmaccu_vv_h, 4)
2100	GEN_VEXT_VV(vwmaccu_vv_w, 8)
2101	GEN_VEXT_VV(vwmacc_vv_b, 2)
2102	GEN_VEXT_VV(vwmacc_vv_h, 4)
2103	GEN_VEXT_VV(vwmacc_vv_w, 8)
2104	GEN_VEXT_VV(vwmaccsu_vv_b, 2)
2105	GEN_VEXT_VV(vwmaccsu_vv_h, 4)
2106	GEN_VEXT_VV(vwmaccsu_vv_w, 8)
2107
2108	RVVCALL(OPIVX3, vwmaccu_vx_b, WOP_UUU_B, H2, H1, DO_MACC)
2109	RVVCALL(OPIVX3, vwmaccu_vx_h, WOP_UUU_H, H4, H2, DO_MACC)
2110	RVVCALL(OPIVX3, vwmaccu_vx_w, WOP_UUU_W, H8, H4, DO_MACC)
2111	RVVCALL(OPIVX3, vwmacc_vx_b, WOP_SSS_B, H2, H1, DO_MACC)
2112	RVVCALL(OPIVX3, vwmacc_vx_h, WOP_SSS_H, H4, H2, DO_MACC)
2113	RVVCALL(OPIVX3, vwmacc_vx_w, WOP_SSS_W, H8, H4, DO_MACC)
2114	RVVCALL(OPIVX3, vwmaccsu_vx_b, WOP_SSU_B, H2, H1, DO_MACC)
2115	RVVCALL(OPIVX3, vwmaccsu_vx_h, WOP_SSU_H, H4, H2, DO_MACC)
2116	RVVCALL(OPIVX3, vwmaccsu_vx_w, WOP_SSU_W, H8, H4, DO_MACC)
2117	RVVCALL(OPIVX3, vwmaccus_vx_b, WOP_SUS_B, H2, H1, DO_MACC)
2118	RVVCALL(OPIVX3, vwmaccus_vx_h, WOP_SUS_H, H4, H2, DO_MACC)
2119	RVVCALL(OPIVX3, vwmaccus_vx_w, WOP_SUS_W, H8, H4, DO_MACC)
2120	GEN_VEXT_VX(vwmaccu_vx_b, 2)
2121	GEN_VEXT_VX(vwmaccu_vx_h, 4)
2122	GEN_VEXT_VX(vwmaccu_vx_w, 8)
2123	GEN_VEXT_VX(vwmacc_vx_b, 2)
2124	GEN_VEXT_VX(vwmacc_vx_h, 4)
2125	GEN_VEXT_VX(vwmacc_vx_w, 8)
2126	GEN_VEXT_VX(vwmaccsu_vx_b, 2)
2127	GEN_VEXT_VX(vwmaccsu_vx_h, 4)
2128	GEN_VEXT_VX(vwmaccsu_vx_w, 8)
2129	GEN_VEXT_VX(vwmaccus_vx_b, 2)
2130	GEN_VEXT_VX(vwmaccus_vx_h, 4)
2131	GEN_VEXT_VX(vwmaccus_vx_w, 8)
2132
2133	/* Vector Integer Merge and Move Instructions */
2134	#define GEN_VEXT_VMV_VV(NAME, ETYPE, H) \
2135	void HELPER(NAME)(void vd, void vs1, CPURISCVState *env, \
2136	uint32_t desc) \
2137	{ \
2138	uint32_t vl = env->vl; \
2139	uint32_t esz = sizeof(ETYPE); \
2140	uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
2141	uint32_t vta = vext_vta(desc); \
2142	uint32_t i; \
2143	\
2144	VSTART_CHECK_EARLY_EXIT(env, vl); \
2145	\
2146	for (i = env->vstart; i < vl; i++) { \
2147	ETYPE s1 = ((ETYPE )vs1 + H(i)); \
2148	((ETYPE )vd + H(i)) = s1; \
2149	} \
2150	env->vstart = 0; \
2151	/* set tail elements to 1s */ \
2152	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
2153	}
2154
2155	GEN_VEXT_VMV_VV(vmv_v_v_b, int8_t, H1)
2156	GEN_VEXT_VMV_VV(vmv_v_v_h, int16_t, H2)
2157	GEN_VEXT_VMV_VV(vmv_v_v_w, int32_t, H4)
2158	GEN_VEXT_VMV_VV(vmv_v_v_d, int64_t, H8)
2159
2160	#define GEN_VEXT_VMV_VX(NAME, ETYPE, H) \
2161	void HELPER(NAME)(void vd, uint64_t s1, CPURISCVState env, \
2162	uint32_t desc) \
2163	{ \
2164	uint32_t vl = env->vl; \
2165	uint32_t esz = sizeof(ETYPE); \
2166	uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
2167	uint32_t vta = vext_vta(desc); \
2168	uint32_t i; \
2169	\
2170	VSTART_CHECK_EARLY_EXIT(env, vl); \
2171	\
2172	for (i = env->vstart; i < vl; i++) { \
2173	((ETYPE )vd + H(i)) = (ETYPE)s1; \
2174	} \
2175	env->vstart = 0; \
2176	/* set tail elements to 1s */ \
2177	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
2178	}
2179
2180	GEN_VEXT_VMV_VX(vmv_v_x_b, int8_t, H1)
2181	GEN_VEXT_VMV_VX(vmv_v_x_h, int16_t, H2)
2182	GEN_VEXT_VMV_VX(vmv_v_x_w, int32_t, H4)
2183	GEN_VEXT_VMV_VX(vmv_v_x_d, int64_t, H8)
2184
2185	#define GEN_VEXT_SET_VELEM0(NAME, ETYPE, H) \
2186	void HELPER(NAME)(void vd, uint64_t s1, CPURISCVState env, \
2187	uint32_t desc) \
2188	{ \
2189	uint32_t esz = sizeof(ETYPE); \
2190	uint32_t vlenb = riscv_cpu_cfg(env)->vlenb; \
2191	uint32_t vta = vext_vta(desc); \
2192	\
2193	((ETYPE )vd + H(0)) = (ETYPE)s1; \
2194	/* Treat every element past vd[0] as tail for scalar-to-vector moves. */ \
2195	vext_set_elems_1s(vd, vta, esz, vlenb); \
2196	}
2197
2198	GEN_VEXT_SET_VELEM0(vset_velem0_b, int8_t, H1)
2199	GEN_VEXT_SET_VELEM0(vset_velem0_h, int16_t, H2)
2200	GEN_VEXT_SET_VELEM0(vset_velem0_w, int32_t, H4)
2201	GEN_VEXT_SET_VELEM0(vset_velem0_d, int64_t, H8)
2202
2203	#define GEN_VEXT_VMERGE_VV(NAME, ETYPE, H) \
2204	void HELPER(NAME)(void vd, void v0, void vs1, void vs2, \
2205	CPURISCVState *env, uint32_t desc) \
2206	{ \
2207	uint32_t vl = env->vl; \
2208	uint32_t esz = sizeof(ETYPE); \
2209	uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
2210	uint32_t vta = vext_vta(desc); \
2211	uint32_t i; \
2212	\
2213	VSTART_CHECK_EARLY_EXIT(env, vl); \
2214	\
2215	for (i = env->vstart; i < vl; i++) { \
2216	ETYPE *vt = (!vext_elem_mask(v0, i) ? vs2 : vs1); \
2217	((ETYPE )vd + H(i)) = *(vt + H(i)); \
2218	} \
2219	env->vstart = 0; \
2220	/* set tail elements to 1s */ \
2221	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
2222	}
2223
2224	GEN_VEXT_VMERGE_VV(vmerge_vvm_b, int8_t, H1)
2225	GEN_VEXT_VMERGE_VV(vmerge_vvm_h, int16_t, H2)
2226	GEN_VEXT_VMERGE_VV(vmerge_vvm_w, int32_t, H4)
2227	GEN_VEXT_VMERGE_VV(vmerge_vvm_d, int64_t, H8)
2228
2229	#define GEN_VEXT_VMERGE_VX(NAME, ETYPE, H) \
2230	void HELPER(NAME)(void vd, void v0, target_ulong s1, \
2231	void vs2, CPURISCVState env, uint32_t desc) \
2232	{ \
2233	uint32_t vl = env->vl; \
2234	uint32_t esz = sizeof(ETYPE); \
2235	uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
2236	uint32_t vta = vext_vta(desc); \
2237	uint32_t i; \
2238	\
2239	VSTART_CHECK_EARLY_EXIT(env, vl); \
2240	\
2241	for (i = env->vstart; i < vl; i++) { \
2242	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
2243	ETYPE d = (!vext_elem_mask(v0, i) ? s2 : \
2244	(ETYPE)(target_long)s1); \
2245	((ETYPE )vd + H(i)) = d; \
2246	} \
2247	env->vstart = 0; \
2248	/* set tail elements to 1s */ \
2249	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
2250	}
2251
2252	GEN_VEXT_VMERGE_VX(vmerge_vxm_b, int8_t, H1)
2253	GEN_VEXT_VMERGE_VX(vmerge_vxm_h, int16_t, H2)
2254	GEN_VEXT_VMERGE_VX(vmerge_vxm_w, int32_t, H4)
2255	GEN_VEXT_VMERGE_VX(vmerge_vxm_d, int64_t, H8)
2256
2257	/*
2258	* Vector Fixed-Point Arithmetic Instructions
2259	*/
2260
2261	/* Vector Single-Width Saturating Add and Subtract */
2262
2263	/*
2264	* As fixed point instructions probably have round mode and saturation,
2265	* define common macros for fixed point here.
2266	*/
2267	typedef void opivv2_rm_fn(void vd, void vs1, void *vs2, int i,
2268	CPURISCVState *env, uint8_t vxrm);
2269
2270	#define OPIVV2_RM(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
2271	static inline void \
2272	do_##NAME(void vd, void vs1, void *vs2, int i, \
2273	CPURISCVState *env, uint8_t vxrm) \
2274	{ \
2275	TX1 s1 = ((T1 )vs1 + HS1(i)); \
2276	TX2 s2 = ((T2 )vs2 + HS2(i)); \
2277	((TD )vd + HD(i)) = OP(env, vxrm, s2, s1); \
2278	}
2279
2280	static inline void
2281	vext_vv_rm_1(void vd, void v0, void vs1, void vs2,
2282	CPURISCVState *env,
2283	uint32_t vl, uint32_t vm, uint8_t vxrm,
2284	opivv2_rm_fn *fn, uint32_t vma, uint32_t esz)
2285	{
2286	for (uint32_t i = env->vstart; i < vl; i++) {
2287	if (!vm && !vext_elem_mask(v0, i)) {
2288	/* set masked-off elements to 1s */
2289	vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
2290	continue;
2291	}
2292	fn(vd, vs1, vs2, i, env, vxrm);
2293	}
2294	env->vstart = 0;
2295	}
2296
2297	static inline void
2298	vext_vv_rm_2(void vd, void v0, void vs1, void vs2,
2299	CPURISCVState *env,
2300	uint32_t desc,
2301	opivv2_rm_fn *fn, uint32_t esz)
2302	{
2303	uint32_t vm = vext_vm(desc);
2304	uint32_t vl = env->vl;
2305	uint32_t total_elems = vext_get_total_elems(env, desc, esz);
2306	uint32_t vta = vext_vta(desc);
2307	uint32_t vma = vext_vma(desc);
2308
2309	VSTART_CHECK_EARLY_EXIT(env, vl);
2310
2311	switch (env->vxrm) {
2312	case 0: /* rnu */
2313	vext_vv_rm_1(vd, v0, vs1, vs2,
2314	env, vl, vm, 0, fn, vma, esz);
2315	break;
2316	case 1: /* rne */
2317	vext_vv_rm_1(vd, v0, vs1, vs2,
2318	env, vl, vm, 1, fn, vma, esz);
2319	break;
2320	case 2: /* rdn */
2321	vext_vv_rm_1(vd, v0, vs1, vs2,
2322	env, vl, vm, 2, fn, vma, esz);
2323	break;
2324	default: /* rod */
2325	vext_vv_rm_1(vd, v0, vs1, vs2,
2326	env, vl, vm, 3, fn, vma, esz);
2327	break;
2328	}
2329	/* set tail elements to 1s */
2330	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
2331	}
2332
2333	/* generate helpers for fixed point instructions with OPIVV format */
2334	#define GEN_VEXT_VV_RM(NAME, ESZ) \
2335	void HELPER(NAME)(void vd, void v0, void vs1, void vs2, \
2336	CPURISCVState *env, uint32_t desc) \
2337	{ \
2338	vext_vv_rm_2(vd, v0, vs1, vs2, env, desc, \
2339	do_##NAME, ESZ); \
2340	}
2341
2342	static inline uint8_t saddu8(CPURISCVState *env, uint8_t vxrm, uint8_t a,
2343	uint8_t b)
2344	{
2345	uint8_t res = a + b;
2346	if (res < a) {
2347	res = UINT8_MAX;
2348	env->vxsat = 0x1;
2349	}
2350	return res;
2351	}
2352
2353	static inline uint16_t saddu16(CPURISCVState *env, uint8_t vxrm, uint16_t a,
2354	uint16_t b)
2355	{
2356	uint16_t res = a + b;
2357	if (res < a) {
2358	res = UINT16_MAX;
2359	env->vxsat = 0x1;
2360	}
2361	return res;
2362	}
2363
2364	static inline uint32_t saddu32(CPURISCVState *env, uint8_t vxrm, uint32_t a,
2365	uint32_t b)
2366	{
2367	uint32_t res = a + b;
2368	if (res < a) {
2369	res = UINT32_MAX;
2370	env->vxsat = 0x1;
2371	}
2372	return res;
2373	}
2374
2375	static inline uint64_t saddu64(CPURISCVState *env, uint8_t vxrm, uint64_t a,
2376	uint64_t b)
2377	{
2378	uint64_t res = a + b;
2379	if (res < a) {
2380	res = UINT64_MAX;
2381	env->vxsat = 0x1;
2382	}
2383	return res;
2384	}
2385
2386	RVVCALL(OPIVV2_RM, vsaddu_vv_b, OP_UUU_B, H1, H1, H1, saddu8)
2387	RVVCALL(OPIVV2_RM, vsaddu_vv_h, OP_UUU_H, H2, H2, H2, saddu16)
2388	RVVCALL(OPIVV2_RM, vsaddu_vv_w, OP_UUU_W, H4, H4, H4, saddu32)
2389	RVVCALL(OPIVV2_RM, vsaddu_vv_d, OP_UUU_D, H8, H8, H8, saddu64)
2390	GEN_VEXT_VV_RM(vsaddu_vv_b, 1)
2391	GEN_VEXT_VV_RM(vsaddu_vv_h, 2)
2392	GEN_VEXT_VV_RM(vsaddu_vv_w, 4)
2393	GEN_VEXT_VV_RM(vsaddu_vv_d, 8)
2394
2395	typedef void opivx2_rm_fn(void vd, target_long s1, void vs2, int i,
2396	CPURISCVState *env, uint8_t vxrm);
2397
2398	#define OPIVX2_RM(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \
2399	static inline void \
2400	do_##NAME(void vd, target_long s1, void vs2, int i, \
2401	CPURISCVState *env, uint8_t vxrm) \
2402	{ \
2403	TX2 s2 = ((T2 )vs2 + HS2(i)); \
2404	((TD )vd + HD(i)) = OP(env, vxrm, s2, (TX1)(T1)s1); \
2405	}
2406
2407	static inline void
2408	vext_vx_rm_1(void vd, void v0, target_long s1, void *vs2,
2409	CPURISCVState *env,
2410	uint32_t vl, uint32_t vm, uint8_t vxrm,
2411	opivx2_rm_fn *fn, uint32_t vma, uint32_t esz)
2412	{
2413	for (uint32_t i = env->vstart; i < vl; i++) {
2414	if (!vm && !vext_elem_mask(v0, i)) {
2415	/* set masked-off elements to 1s */
2416	vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
2417	continue;
2418	}
2419	fn(vd, s1, vs2, i, env, vxrm);
2420	}
2421	env->vstart = 0;
2422	}
2423
2424	static inline void
2425	vext_vx_rm_2(void vd, void v0, target_long s1, void *vs2,
2426	CPURISCVState *env,
2427	uint32_t desc,
2428	opivx2_rm_fn *fn, uint32_t esz)
2429	{
2430	uint32_t vm = vext_vm(desc);
2431	uint32_t vl = env->vl;
2432	uint32_t total_elems = vext_get_total_elems(env, desc, esz);
2433	uint32_t vta = vext_vta(desc);
2434	uint32_t vma = vext_vma(desc);
2435
2436	VSTART_CHECK_EARLY_EXIT(env, vl);
2437
2438	switch (env->vxrm) {
2439	case 0: /* rnu */
2440	vext_vx_rm_1(vd, v0, s1, vs2,
2441	env, vl, vm, 0, fn, vma, esz);
2442	break;
2443	case 1: /* rne */
2444	vext_vx_rm_1(vd, v0, s1, vs2,
2445	env, vl, vm, 1, fn, vma, esz);
2446	break;
2447	case 2: /* rdn */
2448	vext_vx_rm_1(vd, v0, s1, vs2,
2449	env, vl, vm, 2, fn, vma, esz);
2450	break;
2451	default: /* rod */
2452	vext_vx_rm_1(vd, v0, s1, vs2,
2453	env, vl, vm, 3, fn, vma, esz);
2454	break;
2455	}
2456	/* set tail elements to 1s */
2457	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
2458	}
2459
2460	/* generate helpers for fixed point instructions with OPIVX format */
2461	#define GEN_VEXT_VX_RM(NAME, ESZ) \
2462	void HELPER(NAME)(void vd, void v0, target_ulong s1, \
2463	void vs2, CPURISCVState env, \
2464	uint32_t desc) \
2465	{ \
2466	vext_vx_rm_2(vd, v0, s1, vs2, env, desc, \
2467	do_##NAME, ESZ); \
2468	}
2469
2470	RVVCALL(OPIVX2_RM, vsaddu_vx_b, OP_UUU_B, H1, H1, saddu8)
2471	RVVCALL(OPIVX2_RM, vsaddu_vx_h, OP_UUU_H, H2, H2, saddu16)
2472	RVVCALL(OPIVX2_RM, vsaddu_vx_w, OP_UUU_W, H4, H4, saddu32)
2473	RVVCALL(OPIVX2_RM, vsaddu_vx_d, OP_UUU_D, H8, H8, saddu64)
2474	GEN_VEXT_VX_RM(vsaddu_vx_b, 1)
2475	GEN_VEXT_VX_RM(vsaddu_vx_h, 2)
2476	GEN_VEXT_VX_RM(vsaddu_vx_w, 4)
2477	GEN_VEXT_VX_RM(vsaddu_vx_d, 8)
2478
2479	static inline int8_t sadd8(CPURISCVState *env, uint8_t vxrm, int8_t a,
2480	int8_t b)
2481	{
2482	int8_t res = a + b;
2483	if ((res ^ a) & (res ^ b) & INT8_MIN) {
2484	res = a > 0 ? INT8_MAX : INT8_MIN;
2485	env->vxsat = 0x1;
2486	}
2487	return res;
2488	}
2489
2490	static inline int16_t sadd16(CPURISCVState *env, uint8_t vxrm, int16_t a,
2491	int16_t b)
2492	{
2493	int16_t res = a + b;
2494	if ((res ^ a) & (res ^ b) & INT16_MIN) {
2495	res = a > 0 ? INT16_MAX : INT16_MIN;
2496	env->vxsat = 0x1;
2497	}
2498	return res;
2499	}
2500
2501	static inline int32_t sadd32(CPURISCVState *env, uint8_t vxrm, int32_t a,
2502	int32_t b)
2503	{
2504	int32_t res = a + b;
2505	if ((res ^ a) & (res ^ b) & INT32_MIN) {
2506	res = a > 0 ? INT32_MAX : INT32_MIN;
2507	env->vxsat = 0x1;
2508	}
2509	return res;
2510	}
2511
2512	static inline int64_t sadd64(CPURISCVState *env, uint8_t vxrm, int64_t a,
2513	int64_t b)
2514	{
2515	int64_t res = a + b;
2516	if ((res ^ a) & (res ^ b) & INT64_MIN) {
2517	res = a > 0 ? INT64_MAX : INT64_MIN;
2518	env->vxsat = 0x1;
2519	}
2520	return res;
2521	}
2522
2523	RVVCALL(OPIVV2_RM, vsadd_vv_b, OP_SSS_B, H1, H1, H1, sadd8)
2524	RVVCALL(OPIVV2_RM, vsadd_vv_h, OP_SSS_H, H2, H2, H2, sadd16)
2525	RVVCALL(OPIVV2_RM, vsadd_vv_w, OP_SSS_W, H4, H4, H4, sadd32)
2526	RVVCALL(OPIVV2_RM, vsadd_vv_d, OP_SSS_D, H8, H8, H8, sadd64)
2527	GEN_VEXT_VV_RM(vsadd_vv_b, 1)
2528	GEN_VEXT_VV_RM(vsadd_vv_h, 2)
2529	GEN_VEXT_VV_RM(vsadd_vv_w, 4)
2530	GEN_VEXT_VV_RM(vsadd_vv_d, 8)
2531
2532	RVVCALL(OPIVX2_RM, vsadd_vx_b, OP_SSS_B, H1, H1, sadd8)
2533	RVVCALL(OPIVX2_RM, vsadd_vx_h, OP_SSS_H, H2, H2, sadd16)
2534	RVVCALL(OPIVX2_RM, vsadd_vx_w, OP_SSS_W, H4, H4, sadd32)
2535	RVVCALL(OPIVX2_RM, vsadd_vx_d, OP_SSS_D, H8, H8, sadd64)
2536	GEN_VEXT_VX_RM(vsadd_vx_b, 1)
2537	GEN_VEXT_VX_RM(vsadd_vx_h, 2)
2538	GEN_VEXT_VX_RM(vsadd_vx_w, 4)
2539	GEN_VEXT_VX_RM(vsadd_vx_d, 8)
2540
2541	static inline uint8_t ssubu8(CPURISCVState *env, uint8_t vxrm, uint8_t a,
2542	uint8_t b)
2543	{
2544	uint8_t res = a - b;
2545	if (res > a) {
2546	res = 0;
2547	env->vxsat = 0x1;
2548	}
2549	return res;
2550	}
2551
2552	static inline uint16_t ssubu16(CPURISCVState *env, uint8_t vxrm, uint16_t a,
2553	uint16_t b)
2554	{
2555	uint16_t res = a - b;
2556	if (res > a) {
2557	res = 0;
2558	env->vxsat = 0x1;
2559	}
2560	return res;
2561	}
2562
2563	static inline uint32_t ssubu32(CPURISCVState *env, uint8_t vxrm, uint32_t a,
2564	uint32_t b)
2565	{
2566	uint32_t res = a - b;
2567	if (res > a) {
2568	res = 0;
2569	env->vxsat = 0x1;
2570	}
2571	return res;
2572	}
2573
2574	static inline uint64_t ssubu64(CPURISCVState *env, uint8_t vxrm, uint64_t a,
2575	uint64_t b)
2576	{
2577	uint64_t res = a - b;
2578	if (res > a) {
2579	res = 0;
2580	env->vxsat = 0x1;
2581	}
2582	return res;
2583	}
2584
2585	RVVCALL(OPIVV2_RM, vssubu_vv_b, OP_UUU_B, H1, H1, H1, ssubu8)
2586	RVVCALL(OPIVV2_RM, vssubu_vv_h, OP_UUU_H, H2, H2, H2, ssubu16)
2587	RVVCALL(OPIVV2_RM, vssubu_vv_w, OP_UUU_W, H4, H4, H4, ssubu32)
2588	RVVCALL(OPIVV2_RM, vssubu_vv_d, OP_UUU_D, H8, H8, H8, ssubu64)
2589	GEN_VEXT_VV_RM(vssubu_vv_b, 1)
2590	GEN_VEXT_VV_RM(vssubu_vv_h, 2)
2591	GEN_VEXT_VV_RM(vssubu_vv_w, 4)
2592	GEN_VEXT_VV_RM(vssubu_vv_d, 8)
2593
2594	RVVCALL(OPIVX2_RM, vssubu_vx_b, OP_UUU_B, H1, H1, ssubu8)
2595	RVVCALL(OPIVX2_RM, vssubu_vx_h, OP_UUU_H, H2, H2, ssubu16)
2596	RVVCALL(OPIVX2_RM, vssubu_vx_w, OP_UUU_W, H4, H4, ssubu32)
2597	RVVCALL(OPIVX2_RM, vssubu_vx_d, OP_UUU_D, H8, H8, ssubu64)
2598	GEN_VEXT_VX_RM(vssubu_vx_b, 1)
2599	GEN_VEXT_VX_RM(vssubu_vx_h, 2)
2600	GEN_VEXT_VX_RM(vssubu_vx_w, 4)
2601	GEN_VEXT_VX_RM(vssubu_vx_d, 8)
2602
2603	static inline int8_t ssub8(CPURISCVState *env, uint8_t vxrm, int8_t a,
2604	int8_t b)
2605	{
2606	int8_t res = a - b;
2607	if ((res ^ a) & (a ^ b) & INT8_MIN) {
2608	res = a >= 0 ? INT8_MAX : INT8_MIN;
2609	env->vxsat = 0x1;
2610	}
2611	return res;
2612	}
2613
2614	static inline int16_t ssub16(CPURISCVState *env, uint8_t vxrm, int16_t a,
2615	int16_t b)
2616	{
2617	int16_t res = a - b;
2618	if ((res ^ a) & (a ^ b) & INT16_MIN) {
2619	res = a >= 0 ? INT16_MAX : INT16_MIN;
2620	env->vxsat = 0x1;
2621	}
2622	return res;
2623	}
2624
2625	static inline int32_t ssub32(CPURISCVState *env, uint8_t vxrm, int32_t a,
2626	int32_t b)
2627	{
2628	int32_t res = a - b;
2629	if ((res ^ a) & (a ^ b) & INT32_MIN) {
2630	res = a >= 0 ? INT32_MAX : INT32_MIN;
2631	env->vxsat = 0x1;
2632	}
2633	return res;
2634	}
2635
2636	static inline int64_t ssub64(CPURISCVState *env, uint8_t vxrm, int64_t a,
2637	int64_t b)
2638	{
2639	int64_t res = a - b;
2640	if ((res ^ a) & (a ^ b) & INT64_MIN) {
2641	res = a >= 0 ? INT64_MAX : INT64_MIN;
2642	env->vxsat = 0x1;
2643	}
2644	return res;
2645	}
2646
2647	RVVCALL(OPIVV2_RM, vssub_vv_b, OP_SSS_B, H1, H1, H1, ssub8)
2648	RVVCALL(OPIVV2_RM, vssub_vv_h, OP_SSS_H, H2, H2, H2, ssub16)
2649	RVVCALL(OPIVV2_RM, vssub_vv_w, OP_SSS_W, H4, H4, H4, ssub32)
2650	RVVCALL(OPIVV2_RM, vssub_vv_d, OP_SSS_D, H8, H8, H8, ssub64)
2651	GEN_VEXT_VV_RM(vssub_vv_b, 1)
2652	GEN_VEXT_VV_RM(vssub_vv_h, 2)
2653	GEN_VEXT_VV_RM(vssub_vv_w, 4)
2654	GEN_VEXT_VV_RM(vssub_vv_d, 8)
2655
2656	RVVCALL(OPIVX2_RM, vssub_vx_b, OP_SSS_B, H1, H1, ssub8)
2657	RVVCALL(OPIVX2_RM, vssub_vx_h, OP_SSS_H, H2, H2, ssub16)
2658	RVVCALL(OPIVX2_RM, vssub_vx_w, OP_SSS_W, H4, H4, ssub32)
2659	RVVCALL(OPIVX2_RM, vssub_vx_d, OP_SSS_D, H8, H8, ssub64)
2660	GEN_VEXT_VX_RM(vssub_vx_b, 1)
2661	GEN_VEXT_VX_RM(vssub_vx_h, 2)
2662	GEN_VEXT_VX_RM(vssub_vx_w, 4)
2663	GEN_VEXT_VX_RM(vssub_vx_d, 8)
2664
2665	/* Vector Single-Width Averaging Add and Subtract */
2666	static inline uint8_t get_round(uint8_t vxrm, uint64_t v, uint8_t shift)
2667	{
2668	uint8_t d = extract64(v, shift, 1);
2669	uint8_t d1;
2670	uint64_t D1, D2;
2671
2672	if (shift == 0 \|\| shift > 64) {
2673	return 0;
2674	}
2675
2676	d1 = extract64(v, shift - 1, 1);
2677	D1 = extract64(v, 0, shift);
2678	switch (vxrm) {
2679	case 0:
2680	/* round-to-nearest-up (add +0.5 LSB) */
2681	return d1;
2682	case 1:
2683	/* round-to-nearest-even */
2684	if (shift > 1) {
2685	D2 = extract64(v, 0, shift - 1);
2686	return d1 & ((D2 != 0) \| d);
2687	} else {
2688	return d1 & d;
2689	}
2690	case 2:
2691	/* round-down (truncate) */
2692	return 0;
2693	case 3:
2694	/* round-to-odd (OR bits into LSB, aka "jam") */
2695	return !d & (D1 != 0);
2696	default:
2697	g_assert_not_reached();
2698	}
2699	}
2700
2701	static inline int32_t aadd32(CPURISCVState *env, uint8_t vxrm, int32_t a,
2702	int32_t b)
2703	{
2704	int64_t res = (int64_t)a + b;
2705	uint8_t round = get_round(vxrm, res, 1);
2706
2707	return (res >> 1) + round;
2708	}
2709
2710	static inline int64_t aadd64(CPURISCVState *env, uint8_t vxrm, int64_t a,
2711	int64_t b)
2712	{
2713	int64_t res = a + b;
2714	uint8_t round = get_round(vxrm, res, 1);
2715	int64_t over = (res ^ a) & (res ^ b) & INT64_MIN;
2716
2717	/* With signed overflow, bit 64 is inverse of bit 63. */
2718	return ((res >> 1) ^ over) + round;
2719	}
2720
2721	RVVCALL(OPIVV2_RM, vaadd_vv_b, OP_SSS_B, H1, H1, H1, aadd32)
2722	RVVCALL(OPIVV2_RM, vaadd_vv_h, OP_SSS_H, H2, H2, H2, aadd32)
2723	RVVCALL(OPIVV2_RM, vaadd_vv_w, OP_SSS_W, H4, H4, H4, aadd32)
2724	RVVCALL(OPIVV2_RM, vaadd_vv_d, OP_SSS_D, H8, H8, H8, aadd64)
2725	GEN_VEXT_VV_RM(vaadd_vv_b, 1)
2726	GEN_VEXT_VV_RM(vaadd_vv_h, 2)
2727	GEN_VEXT_VV_RM(vaadd_vv_w, 4)
2728	GEN_VEXT_VV_RM(vaadd_vv_d, 8)
2729
2730	RVVCALL(OPIVX2_RM, vaadd_vx_b, OP_SSS_B, H1, H1, aadd32)
2731	RVVCALL(OPIVX2_RM, vaadd_vx_h, OP_SSS_H, H2, H2, aadd32)
2732	RVVCALL(OPIVX2_RM, vaadd_vx_w, OP_SSS_W, H4, H4, aadd32)
2733	RVVCALL(OPIVX2_RM, vaadd_vx_d, OP_SSS_D, H8, H8, aadd64)
2734	GEN_VEXT_VX_RM(vaadd_vx_b, 1)
2735	GEN_VEXT_VX_RM(vaadd_vx_h, 2)
2736	GEN_VEXT_VX_RM(vaadd_vx_w, 4)
2737	GEN_VEXT_VX_RM(vaadd_vx_d, 8)
2738
2739	static inline uint32_t aaddu32(CPURISCVState *env, uint8_t vxrm,
2740	uint32_t a, uint32_t b)
2741	{
2742	uint64_t res = (uint64_t)a + b;
2743	uint8_t round = get_round(vxrm, res, 1);
2744
2745	return (res >> 1) + round;
2746	}
2747
2748	static inline uint64_t aaddu64(CPURISCVState *env, uint8_t vxrm,
2749	uint64_t a, uint64_t b)
2750	{
2751	uint64_t res = a + b;
2752	uint8_t round = get_round(vxrm, res, 1);
2753	uint64_t over = (uint64_t)(res < a) << 63;
2754
2755	return ((res >> 1) \| over) + round;
2756	}
2757
2758	RVVCALL(OPIVV2_RM, vaaddu_vv_b, OP_UUU_B, H1, H1, H1, aaddu32)
2759	RVVCALL(OPIVV2_RM, vaaddu_vv_h, OP_UUU_H, H2, H2, H2, aaddu32)
2760	RVVCALL(OPIVV2_RM, vaaddu_vv_w, OP_UUU_W, H4, H4, H4, aaddu32)
2761	RVVCALL(OPIVV2_RM, vaaddu_vv_d, OP_UUU_D, H8, H8, H8, aaddu64)
2762	GEN_VEXT_VV_RM(vaaddu_vv_b, 1)
2763	GEN_VEXT_VV_RM(vaaddu_vv_h, 2)
2764	GEN_VEXT_VV_RM(vaaddu_vv_w, 4)
2765	GEN_VEXT_VV_RM(vaaddu_vv_d, 8)
2766
2767	RVVCALL(OPIVX2_RM, vaaddu_vx_b, OP_UUU_B, H1, H1, aaddu32)
2768	RVVCALL(OPIVX2_RM, vaaddu_vx_h, OP_UUU_H, H2, H2, aaddu32)
2769	RVVCALL(OPIVX2_RM, vaaddu_vx_w, OP_UUU_W, H4, H4, aaddu32)
2770	RVVCALL(OPIVX2_RM, vaaddu_vx_d, OP_UUU_D, H8, H8, aaddu64)
2771	GEN_VEXT_VX_RM(vaaddu_vx_b, 1)
2772	GEN_VEXT_VX_RM(vaaddu_vx_h, 2)
2773	GEN_VEXT_VX_RM(vaaddu_vx_w, 4)
2774	GEN_VEXT_VX_RM(vaaddu_vx_d, 8)
2775
2776	static inline int32_t asub32(CPURISCVState *env, uint8_t vxrm, int32_t a,
2777	int32_t b)
2778	{
2779	int64_t res = (int64_t)a - b;
2780	uint8_t round = get_round(vxrm, res, 1);
2781
2782	return (res >> 1) + round;
2783	}
2784
2785	static inline int64_t asub64(CPURISCVState *env, uint8_t vxrm, int64_t a,
2786	int64_t b)
2787	{
2788	int64_t res = (int64_t)a - b;
2789	uint8_t round = get_round(vxrm, res, 1);
2790	int64_t over = (res ^ a) & (a ^ b) & INT64_MIN;
2791
2792	/* With signed overflow, bit 64 is inverse of bit 63. */
2793	return ((res >> 1) ^ over) + round;
2794	}
2795
2796	RVVCALL(OPIVV2_RM, vasub_vv_b, OP_SSS_B, H1, H1, H1, asub32)
2797	RVVCALL(OPIVV2_RM, vasub_vv_h, OP_SSS_H, H2, H2, H2, asub32)
2798	RVVCALL(OPIVV2_RM, vasub_vv_w, OP_SSS_W, H4, H4, H4, asub32)
2799	RVVCALL(OPIVV2_RM, vasub_vv_d, OP_SSS_D, H8, H8, H8, asub64)
2800	GEN_VEXT_VV_RM(vasub_vv_b, 1)
2801	GEN_VEXT_VV_RM(vasub_vv_h, 2)
2802	GEN_VEXT_VV_RM(vasub_vv_w, 4)
2803	GEN_VEXT_VV_RM(vasub_vv_d, 8)
2804
2805	RVVCALL(OPIVX2_RM, vasub_vx_b, OP_SSS_B, H1, H1, asub32)
2806	RVVCALL(OPIVX2_RM, vasub_vx_h, OP_SSS_H, H2, H2, asub32)
2807	RVVCALL(OPIVX2_RM, vasub_vx_w, OP_SSS_W, H4, H4, asub32)
2808	RVVCALL(OPIVX2_RM, vasub_vx_d, OP_SSS_D, H8, H8, asub64)
2809	GEN_VEXT_VX_RM(vasub_vx_b, 1)
2810	GEN_VEXT_VX_RM(vasub_vx_h, 2)
2811	GEN_VEXT_VX_RM(vasub_vx_w, 4)
2812	GEN_VEXT_VX_RM(vasub_vx_d, 8)
2813
2814	static inline uint32_t asubu32(CPURISCVState *env, uint8_t vxrm,
2815	uint32_t a, uint32_t b)
2816	{
2817	int64_t res = (int64_t)a - b;
2818	uint8_t round = get_round(vxrm, res, 1);
2819
2820	return (res >> 1) + round;
2821	}
2822
2823	static inline uint64_t asubu64(CPURISCVState *env, uint8_t vxrm,
2824	uint64_t a, uint64_t b)
2825	{
2826	uint64_t res = (uint64_t)a - b;
2827	uint8_t round = get_round(vxrm, res, 1);
2828	uint64_t over = (uint64_t)(res > a) << 63;
2829
2830	return ((res >> 1) \| over) + round;
2831	}
2832
2833	RVVCALL(OPIVV2_RM, vasubu_vv_b, OP_UUU_B, H1, H1, H1, asubu32)
2834	RVVCALL(OPIVV2_RM, vasubu_vv_h, OP_UUU_H, H2, H2, H2, asubu32)
2835	RVVCALL(OPIVV2_RM, vasubu_vv_w, OP_UUU_W, H4, H4, H4, asubu32)
2836	RVVCALL(OPIVV2_RM, vasubu_vv_d, OP_UUU_D, H8, H8, H8, asubu64)
2837	GEN_VEXT_VV_RM(vasubu_vv_b, 1)
2838	GEN_VEXT_VV_RM(vasubu_vv_h, 2)
2839	GEN_VEXT_VV_RM(vasubu_vv_w, 4)
2840	GEN_VEXT_VV_RM(vasubu_vv_d, 8)
2841
2842	RVVCALL(OPIVX2_RM, vasubu_vx_b, OP_UUU_B, H1, H1, asubu32)
2843	RVVCALL(OPIVX2_RM, vasubu_vx_h, OP_UUU_H, H2, H2, asubu32)
2844	RVVCALL(OPIVX2_RM, vasubu_vx_w, OP_UUU_W, H4, H4, asubu32)
2845	RVVCALL(OPIVX2_RM, vasubu_vx_d, OP_UUU_D, H8, H8, asubu64)
2846	GEN_VEXT_VX_RM(vasubu_vx_b, 1)
2847	GEN_VEXT_VX_RM(vasubu_vx_h, 2)
2848	GEN_VEXT_VX_RM(vasubu_vx_w, 4)
2849	GEN_VEXT_VX_RM(vasubu_vx_d, 8)
2850
2851	/* Vector Single-Width Fractional Multiply with Rounding and Saturation */
2852	static inline int8_t vsmul8(CPURISCVState *env, uint8_t vxrm, int8_t a,
2853	int8_t b)
2854	{
2855	uint8_t round;
2856	int16_t res;
2857
2858	res = (int16_t)a * (int16_t)b;
2859	round = get_round(vxrm, res, 7);
2860	res = (res >> 7) + round;
2861
2862	if (res > INT8_MAX) {
2863	env->vxsat = 0x1;
2864	return INT8_MAX;
2865	} else if (res < INT8_MIN) {
2866	env->vxsat = 0x1;
2867	return INT8_MIN;
2868	} else {
2869	return res;
2870	}
2871	}
2872
2873	static int16_t vsmul16(CPURISCVState *env, uint8_t vxrm, int16_t a, int16_t b)
2874	{
2875	uint8_t round;
2876	int32_t res;
2877
2878	res = (int32_t)a * (int32_t)b;
2879	round = get_round(vxrm, res, 15);
2880	res = (res >> 15) + round;
2881
2882	if (res > INT16_MAX) {
2883	env->vxsat = 0x1;
2884	return INT16_MAX;
2885	} else if (res < INT16_MIN) {
2886	env->vxsat = 0x1;
2887	return INT16_MIN;
2888	} else {
2889	return res;
2890	}
2891	}
2892
2893	static int32_t vsmul32(CPURISCVState *env, uint8_t vxrm, int32_t a, int32_t b)
2894	{
2895	uint8_t round;
2896	int64_t res;
2897
2898	res = (int64_t)a * (int64_t)b;
2899	round = get_round(vxrm, res, 31);
2900	res = (res >> 31) + round;
2901
2902	if (res > INT32_MAX) {
2903	env->vxsat = 0x1;
2904	return INT32_MAX;
2905	} else if (res < INT32_MIN) {
2906	env->vxsat = 0x1;
2907	return INT32_MIN;
2908	} else {
2909	return res;
2910	}
2911	}
2912
2913	static int64_t vsmul64(CPURISCVState *env, uint8_t vxrm, int64_t a, int64_t b)
2914	{
2915	uint8_t round;
2916	uint64_t hi_64, lo_64;
2917	int64_t res;
2918
2919	if (a == INT64_MIN && b == INT64_MIN) {
2920	env->vxsat = 1;
2921	return INT64_MAX;
2922	}
2923
2924	muls64(&lo_64, &hi_64, a, b);
2925	round = get_round(vxrm, lo_64, 63);
2926	/*
2927	* Cannot overflow, as there are always
2928	* 2 sign bits after multiply.
2929	*/
2930	res = (hi_64 << 1) \| (lo_64 >> 63);
2931	if (round) {
2932	if (res == INT64_MAX) {
2933	env->vxsat = 1;
2934	} else {
2935	res += 1;
2936	}
2937	}
2938	return res;
2939	}
2940
2941	RVVCALL(OPIVV2_RM, vsmul_vv_b, OP_SSS_B, H1, H1, H1, vsmul8)
2942	RVVCALL(OPIVV2_RM, vsmul_vv_h, OP_SSS_H, H2, H2, H2, vsmul16)
2943	RVVCALL(OPIVV2_RM, vsmul_vv_w, OP_SSS_W, H4, H4, H4, vsmul32)
2944	RVVCALL(OPIVV2_RM, vsmul_vv_d, OP_SSS_D, H8, H8, H8, vsmul64)
2945	GEN_VEXT_VV_RM(vsmul_vv_b, 1)
2946	GEN_VEXT_VV_RM(vsmul_vv_h, 2)
2947	GEN_VEXT_VV_RM(vsmul_vv_w, 4)
2948	GEN_VEXT_VV_RM(vsmul_vv_d, 8)
2949
2950	RVVCALL(OPIVX2_RM, vsmul_vx_b, OP_SSS_B, H1, H1, vsmul8)
2951	RVVCALL(OPIVX2_RM, vsmul_vx_h, OP_SSS_H, H2, H2, vsmul16)
2952	RVVCALL(OPIVX2_RM, vsmul_vx_w, OP_SSS_W, H4, H4, vsmul32)
2953	RVVCALL(OPIVX2_RM, vsmul_vx_d, OP_SSS_D, H8, H8, vsmul64)
2954	GEN_VEXT_VX_RM(vsmul_vx_b, 1)
2955	GEN_VEXT_VX_RM(vsmul_vx_h, 2)
2956	GEN_VEXT_VX_RM(vsmul_vx_w, 4)
2957	GEN_VEXT_VX_RM(vsmul_vx_d, 8)
2958
2959	/* Vector Single-Width Scaling Shift Instructions */
2960	static inline uint8_t
2961	vssrl8(CPURISCVState *env, uint8_t vxrm, uint8_t a, uint8_t b)
2962	{
2963	uint8_t round, shift = b & 0x7;
2964	uint8_t res;
2965
2966	round = get_round(vxrm, a, shift);
2967	res = (a >> shift) + round;
2968	return res;
2969	}
2970	static inline uint16_t
2971	vssrl16(CPURISCVState *env, uint8_t vxrm, uint16_t a, uint16_t b)
2972	{
2973	uint8_t round, shift = b & 0xf;
2974
2975	round = get_round(vxrm, a, shift);
2976	return (a >> shift) + round;
2977	}
2978	static inline uint32_t
2979	vssrl32(CPURISCVState *env, uint8_t vxrm, uint32_t a, uint32_t b)
2980	{
2981	uint8_t round, shift = b & 0x1f;
2982
2983	round = get_round(vxrm, a, shift);
2984	return (a >> shift) + round;
2985	}
2986	static inline uint64_t
2987	vssrl64(CPURISCVState *env, uint8_t vxrm, uint64_t a, uint64_t b)
2988	{
2989	uint8_t round, shift = b & 0x3f;
2990
2991	round = get_round(vxrm, a, shift);
2992	return (a >> shift) + round;
2993	}
2994	RVVCALL(OPIVV2_RM, vssrl_vv_b, OP_UUU_B, H1, H1, H1, vssrl8)
2995	RVVCALL(OPIVV2_RM, vssrl_vv_h, OP_UUU_H, H2, H2, H2, vssrl16)
2996	RVVCALL(OPIVV2_RM, vssrl_vv_w, OP_UUU_W, H4, H4, H4, vssrl32)
2997	RVVCALL(OPIVV2_RM, vssrl_vv_d, OP_UUU_D, H8, H8, H8, vssrl64)
2998	GEN_VEXT_VV_RM(vssrl_vv_b, 1)
2999	GEN_VEXT_VV_RM(vssrl_vv_h, 2)
3000	GEN_VEXT_VV_RM(vssrl_vv_w, 4)
3001	GEN_VEXT_VV_RM(vssrl_vv_d, 8)
3002
3003	RVVCALL(OPIVX2_RM, vssrl_vx_b, OP_UUU_B, H1, H1, vssrl8)
3004	RVVCALL(OPIVX2_RM, vssrl_vx_h, OP_UUU_H, H2, H2, vssrl16)
3005	RVVCALL(OPIVX2_RM, vssrl_vx_w, OP_UUU_W, H4, H4, vssrl32)
3006	RVVCALL(OPIVX2_RM, vssrl_vx_d, OP_UUU_D, H8, H8, vssrl64)
3007	GEN_VEXT_VX_RM(vssrl_vx_b, 1)
3008	GEN_VEXT_VX_RM(vssrl_vx_h, 2)
3009	GEN_VEXT_VX_RM(vssrl_vx_w, 4)
3010	GEN_VEXT_VX_RM(vssrl_vx_d, 8)
3011
3012	static inline int8_t
3013	vssra8(CPURISCVState *env, uint8_t vxrm, int8_t a, int8_t b)
3014	{
3015	uint8_t round, shift = b & 0x7;
3016
3017	round = get_round(vxrm, a, shift);
3018	return (a >> shift) + round;
3019	}
3020	static inline int16_t
3021	vssra16(CPURISCVState *env, uint8_t vxrm, int16_t a, int16_t b)
3022	{
3023	uint8_t round, shift = b & 0xf;
3024
3025	round = get_round(vxrm, a, shift);
3026	return (a >> shift) + round;
3027	}
3028	static inline int32_t
3029	vssra32(CPURISCVState *env, uint8_t vxrm, int32_t a, int32_t b)
3030	{
3031	uint8_t round, shift = b & 0x1f;
3032
3033	round = get_round(vxrm, a, shift);
3034	return (a >> shift) + round;
3035	}
3036	static inline int64_t
3037	vssra64(CPURISCVState *env, uint8_t vxrm, int64_t a, int64_t b)
3038	{
3039	uint8_t round, shift = b & 0x3f;
3040
3041	round = get_round(vxrm, a, shift);
3042	return (a >> shift) + round;
3043	}
3044
3045	RVVCALL(OPIVV2_RM, vssra_vv_b, OP_SSS_B, H1, H1, H1, vssra8)
3046	RVVCALL(OPIVV2_RM, vssra_vv_h, OP_SSS_H, H2, H2, H2, vssra16)
3047	RVVCALL(OPIVV2_RM, vssra_vv_w, OP_SSS_W, H4, H4, H4, vssra32)
3048	RVVCALL(OPIVV2_RM, vssra_vv_d, OP_SSS_D, H8, H8, H8, vssra64)
3049	GEN_VEXT_VV_RM(vssra_vv_b, 1)
3050	GEN_VEXT_VV_RM(vssra_vv_h, 2)
3051	GEN_VEXT_VV_RM(vssra_vv_w, 4)
3052	GEN_VEXT_VV_RM(vssra_vv_d, 8)
3053
3054	RVVCALL(OPIVX2_RM, vssra_vx_b, OP_SSS_B, H1, H1, vssra8)
3055	RVVCALL(OPIVX2_RM, vssra_vx_h, OP_SSS_H, H2, H2, vssra16)
3056	RVVCALL(OPIVX2_RM, vssra_vx_w, OP_SSS_W, H4, H4, vssra32)
3057	RVVCALL(OPIVX2_RM, vssra_vx_d, OP_SSS_D, H8, H8, vssra64)
3058	GEN_VEXT_VX_RM(vssra_vx_b, 1)
3059	GEN_VEXT_VX_RM(vssra_vx_h, 2)
3060	GEN_VEXT_VX_RM(vssra_vx_w, 4)
3061	GEN_VEXT_VX_RM(vssra_vx_d, 8)
3062
3063	/* Vector Narrowing Fixed-Point Clip Instructions */
3064	static inline int8_t
3065	vnclip8(CPURISCVState *env, uint8_t vxrm, int16_t a, int8_t b)
3066	{
3067	uint8_t round, shift = b & 0xf;
3068	int16_t res;
3069
3070	round = get_round(vxrm, a, shift);
3071	res = (a >> shift) + round;
3072	if (res > INT8_MAX) {
3073	env->vxsat = 0x1;
3074	return INT8_MAX;
3075	} else if (res < INT8_MIN) {
3076	env->vxsat = 0x1;
3077	return INT8_MIN;
3078	} else {
3079	return res;
3080	}
3081	}
3082
3083	static inline int16_t
3084	vnclip16(CPURISCVState *env, uint8_t vxrm, int32_t a, int16_t b)
3085	{
3086	uint8_t round, shift = b & 0x1f;
3087	int32_t res;
3088
3089	round = get_round(vxrm, a, shift);
3090	res = (a >> shift) + round;
3091	if (res > INT16_MAX) {
3092	env->vxsat = 0x1;
3093	return INT16_MAX;
3094	} else if (res < INT16_MIN) {
3095	env->vxsat = 0x1;
3096	return INT16_MIN;
3097	} else {
3098	return res;
3099	}
3100	}
3101
3102	static inline int32_t
3103	vnclip32(CPURISCVState *env, uint8_t vxrm, int64_t a, int32_t b)
3104	{
3105	uint8_t round, shift = b & 0x3f;
3106	int64_t res;
3107
3108	round = get_round(vxrm, a, shift);
3109	res = (a >> shift) + round;
3110	if (res > INT32_MAX) {
3111	env->vxsat = 0x1;
3112	return INT32_MAX;
3113	} else if (res < INT32_MIN) {
3114	env->vxsat = 0x1;
3115	return INT32_MIN;
3116	} else {
3117	return res;
3118	}
3119	}
3120
3121	RVVCALL(OPIVV2_RM, vnclip_wv_b, NOP_SSS_B, H1, H2, H1, vnclip8)
3122	RVVCALL(OPIVV2_RM, vnclip_wv_h, NOP_SSS_H, H2, H4, H2, vnclip16)
3123	RVVCALL(OPIVV2_RM, vnclip_wv_w, NOP_SSS_W, H4, H8, H4, vnclip32)
3124	GEN_VEXT_VV_RM(vnclip_wv_b, 1)
3125	GEN_VEXT_VV_RM(vnclip_wv_h, 2)
3126	GEN_VEXT_VV_RM(vnclip_wv_w, 4)
3127
3128	RVVCALL(OPIVX2_RM, vnclip_wx_b, NOP_SSS_B, H1, H2, vnclip8)
3129	RVVCALL(OPIVX2_RM, vnclip_wx_h, NOP_SSS_H, H2, H4, vnclip16)
3130	RVVCALL(OPIVX2_RM, vnclip_wx_w, NOP_SSS_W, H4, H8, vnclip32)
3131	GEN_VEXT_VX_RM(vnclip_wx_b, 1)
3132	GEN_VEXT_VX_RM(vnclip_wx_h, 2)
3133	GEN_VEXT_VX_RM(vnclip_wx_w, 4)
3134
3135	static inline uint8_t
3136	vnclipu8(CPURISCVState *env, uint8_t vxrm, uint16_t a, uint8_t b)
3137	{
3138	uint8_t round, shift = b & 0xf;
3139	uint16_t res;
3140
3141	round = get_round(vxrm, a, shift);
3142	res = (a >> shift) + round;
3143	if (res > UINT8_MAX) {
3144	env->vxsat = 0x1;
3145	return UINT8_MAX;
3146	} else {
3147	return res;
3148	}
3149	}
3150
3151	static inline uint16_t
3152	vnclipu16(CPURISCVState *env, uint8_t vxrm, uint32_t a, uint16_t b)
3153	{
3154	uint8_t round, shift = b & 0x1f;
3155	uint32_t res;
3156
3157	round = get_round(vxrm, a, shift);
3158	res = (a >> shift) + round;
3159	if (res > UINT16_MAX) {
3160	env->vxsat = 0x1;
3161	return UINT16_MAX;
3162	} else {
3163	return res;
3164	}
3165	}
3166
3167	static inline uint32_t
3168	vnclipu32(CPURISCVState *env, uint8_t vxrm, uint64_t a, uint32_t b)
3169	{
3170	uint8_t round, shift = b & 0x3f;
3171	uint64_t res;
3172
3173	round = get_round(vxrm, a, shift);
3174	res = (a >> shift) + round;
3175	if (res > UINT32_MAX) {
3176	env->vxsat = 0x1;
3177	return UINT32_MAX;
3178	} else {
3179	return res;
3180	}
3181	}
3182
3183	RVVCALL(OPIVV2_RM, vnclipu_wv_b, NOP_UUU_B, H1, H2, H1, vnclipu8)
3184	RVVCALL(OPIVV2_RM, vnclipu_wv_h, NOP_UUU_H, H2, H4, H2, vnclipu16)
3185	RVVCALL(OPIVV2_RM, vnclipu_wv_w, NOP_UUU_W, H4, H8, H4, vnclipu32)
3186	GEN_VEXT_VV_RM(vnclipu_wv_b, 1)
3187	GEN_VEXT_VV_RM(vnclipu_wv_h, 2)
3188	GEN_VEXT_VV_RM(vnclipu_wv_w, 4)
3189
3190	RVVCALL(OPIVX2_RM, vnclipu_wx_b, NOP_UUU_B, H1, H2, vnclipu8)
3191	RVVCALL(OPIVX2_RM, vnclipu_wx_h, NOP_UUU_H, H2, H4, vnclipu16)
3192	RVVCALL(OPIVX2_RM, vnclipu_wx_w, NOP_UUU_W, H4, H8, vnclipu32)
3193	GEN_VEXT_VX_RM(vnclipu_wx_b, 1)
3194	GEN_VEXT_VX_RM(vnclipu_wx_h, 2)
3195	GEN_VEXT_VX_RM(vnclipu_wx_w, 4)
3196
3197	/*
3198	* Vector Float Point Arithmetic Instructions
3199	*/
3200	/* Vector Single-Width Floating-Point Add/Subtract Instructions */
3201	#define OPFVV2(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
3202	static void do_##NAME(void vd, void vs1, void *vs2, int i, \
3203	CPURISCVState *env) \
3204	{ \
3205	TX1 s1 = ((T1 )vs1 + HS1(i)); \
3206	TX2 s2 = ((T2 )vs2 + HS2(i)); \
3207	((TD )vd + HD(i)) = OP(s2, s1, &env->fp_status); \
3208	}
3209
3210	#define GEN_VEXT_VV_ENV(NAME, ESZ) \
3211	void HELPER(NAME)(void vd, void v0, void vs1, void vs2, \
3212	CPURISCVState *env, uint32_t desc) \
3213	{ \
3214	uint32_t vm = vext_vm(desc); \
3215	uint32_t vl = env->vl; \
3216	uint32_t total_elems = \
3217	vext_get_total_elems(env, desc, ESZ); \
3218	uint32_t vta = vext_vta(desc); \
3219	uint32_t vma = vext_vma(desc); \
3220	uint32_t i; \
3221	FloatExceptionFlags pre_fflag = \
3222	get_float_exception_flags(&env->fp_status); \
3223	\
3224	VSTART_CHECK_EARLY_EXIT(env, vl); \
3225	\
3226	for (i = env->vstart; i < vl; i++) { \
3227	if (!vm && !vext_elem_mask(v0, i)) { \
3228	/* set masked-off elements to 1s */ \
3229	vext_set_elems_1s(vd, vma, i * ESZ, \
3230	(i + 1) * ESZ); \
3231	continue; \
3232	} \
3233	do_##NAME(vd, vs1, vs2, i, env); \
3234	} \
3235	env->vstart = 0; \
3236	/* set tail elements to 1s */ \
3237	vext_set_elems_1s(vd, vta, vl * ESZ, \
3238	total_elems * ESZ); \
3239	riscv_cpu_check_fflags(env, pre_fflag); \
3240	}
3241
3242	RVVCALL(OPFVV2, vfadd_vv_h_bf16, OP_UUU_H, H2, H2, H2, bfloat16_add)
3243	RVVCALL(OPFVV2, vfadd_vv_h, OP_UUU_H, H2, H2, H2, float16_add)
3244	RVVCALL(OPFVV2, vfadd_vv_w, OP_UUU_W, H4, H4, H4, float32_add)
3245	RVVCALL(OPFVV2, vfadd_vv_d, OP_UUU_D, H8, H8, H8, float64_add)
3246	GEN_VEXT_VV_ENV(vfadd_vv_h_bf16, 2)
3247	GEN_VEXT_VV_ENV(vfadd_vv_h, 2)
3248	GEN_VEXT_VV_ENV(vfadd_vv_w, 4)
3249	GEN_VEXT_VV_ENV(vfadd_vv_d, 8)
3250
3251	#define OPFVF2(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \
3252	static void do_##NAME(void vd, uint64_t s1, void vs2, int i, \
3253	CPURISCVState *env) \
3254	{ \
3255	TX2 s2 = ((T2 )vs2 + HS2(i)); \
3256	((TD )vd + HD(i)) = OP(s2, (TX1)(T1)s1, &env->fp_status);\
3257	}
3258
3259	#define GEN_VEXT_VF(NAME, ESZ) \
3260	void HELPER(NAME)(void vd, void v0, uint64_t s1, void *vs2, \
3261	CPURISCVState *env, uint32_t desc) \
3262	{ \
3263	uint32_t vm = vext_vm(desc); \
3264	uint32_t vl = env->vl; \
3265	uint32_t total_elems = \
3266	vext_get_total_elems(env, desc, ESZ); \
3267	uint32_t vta = vext_vta(desc); \
3268	uint32_t vma = vext_vma(desc); \
3269	uint32_t i; \
3270	FloatExceptionFlags pre_fflag = \
3271	get_float_exception_flags(&env->fp_status); \
3272	\
3273	VSTART_CHECK_EARLY_EXIT(env, vl); \
3274	\
3275	for (i = env->vstart; i < vl; i++) { \
3276	if (!vm && !vext_elem_mask(v0, i)) { \
3277	/* set masked-off elements to 1s */ \
3278	vext_set_elems_1s(vd, vma, i * ESZ, \
3279	(i + 1) * ESZ); \
3280	continue; \
3281	} \
3282	do_##NAME(vd, s1, vs2, i, env); \
3283	} \
3284	env->vstart = 0; \
3285	/* set tail elements to 1s */ \
3286	vext_set_elems_1s(vd, vta, vl * ESZ, \
3287	total_elems * ESZ); \
3288	riscv_cpu_check_fflags(env, pre_fflag); \
3289	}
3290
3291	RVVCALL(OPFVF2, vfadd_vf_h_bf16, OP_UUU_H, H2, H2, bfloat16_add)
3292	RVVCALL(OPFVF2, vfadd_vf_h, OP_UUU_H, H2, H2, float16_add)
3293	RVVCALL(OPFVF2, vfadd_vf_w, OP_UUU_W, H4, H4, float32_add)
3294	RVVCALL(OPFVF2, vfadd_vf_d, OP_UUU_D, H8, H8, float64_add)
3295	GEN_VEXT_VF(vfadd_vf_h_bf16, 2)
3296	GEN_VEXT_VF(vfadd_vf_h, 2)
3297	GEN_VEXT_VF(vfadd_vf_w, 4)
3298	GEN_VEXT_VF(vfadd_vf_d, 8)
3299
3300	RVVCALL(OPFVV2, vfsub_vv_h_bf16, OP_UUU_H, H2, H2, H2, bfloat16_sub)
3301	RVVCALL(OPFVV2, vfsub_vv_h, OP_UUU_H, H2, H2, H2, float16_sub)
3302	RVVCALL(OPFVV2, vfsub_vv_w, OP_UUU_W, H4, H4, H4, float32_sub)
3303	RVVCALL(OPFVV2, vfsub_vv_d, OP_UUU_D, H8, H8, H8, float64_sub)
3304	GEN_VEXT_VV_ENV(vfsub_vv_h_bf16, 2)
3305	GEN_VEXT_VV_ENV(vfsub_vv_h, 2)
3306	GEN_VEXT_VV_ENV(vfsub_vv_w, 4)
3307	GEN_VEXT_VV_ENV(vfsub_vv_d, 8)
3308	RVVCALL(OPFVF2, vfsub_vf_h_bf16, OP_UUU_H, H2, H2, bfloat16_sub)
3309	RVVCALL(OPFVF2, vfsub_vf_h, OP_UUU_H, H2, H2, float16_sub)
3310	RVVCALL(OPFVF2, vfsub_vf_w, OP_UUU_W, H4, H4, float32_sub)
3311	RVVCALL(OPFVF2, vfsub_vf_d, OP_UUU_D, H8, H8, float64_sub)
3312	GEN_VEXT_VF(vfsub_vf_h_bf16, 2)
3313	GEN_VEXT_VF(vfsub_vf_h, 2)
3314	GEN_VEXT_VF(vfsub_vf_w, 4)
3315	GEN_VEXT_VF(vfsub_vf_d, 8)
3316
3317	static uint16_t bfloat16_rsub(uint16_t a, uint16_t b, float_status * s)
3318	{
3319	return bfloat16_sub(b, a, s);
3320	}
3321
3322	static uint16_t float16_rsub(uint16_t a, uint16_t b, float_status *s)
3323	{
3324	return float16_sub(b, a, s);
3325	}
3326
3327	static uint32_t float32_rsub(uint32_t a, uint32_t b, float_status *s)
3328	{
3329	return float32_sub(b, a, s);
3330	}
3331
3332	static uint64_t float64_rsub(uint64_t a, uint64_t b, float_status *s)
3333	{
3334	return float64_sub(b, a, s);
3335	}
3336
3337	RVVCALL(OPFVF2, vfrsub_vf_h_bf16, OP_UUU_H, H2, H2, bfloat16_rsub)
3338	RVVCALL(OPFVF2, vfrsub_vf_h, OP_UUU_H, H2, H2, float16_rsub)
3339	RVVCALL(OPFVF2, vfrsub_vf_w, OP_UUU_W, H4, H4, float32_rsub)
3340	RVVCALL(OPFVF2, vfrsub_vf_d, OP_UUU_D, H8, H8, float64_rsub)
3341	GEN_VEXT_VF(vfrsub_vf_h_bf16, 2)
3342	GEN_VEXT_VF(vfrsub_vf_h, 2)
3343	GEN_VEXT_VF(vfrsub_vf_w, 4)
3344	GEN_VEXT_VF(vfrsub_vf_d, 8)
3345
3346	/* Vector Widening Floating-Point Add/Subtract Instructions */
3347	static uint32_t vfwadd16_bf16(uint16_t a, uint16_t b, float_status *s)
3348	{
3349	return float32_add(bfloat16_to_float32(a, s),
3350	bfloat16_to_float32(b, s), s);
3351	}
3352
3353	static uint32_t vfwadd16(uint16_t a, uint16_t b, float_status *s)
3354	{
3355	return float32_add(float16_to_float32(a, true, s),
3356	float16_to_float32(b, true, s), s);
3357	}
3358
3359	static uint64_t vfwadd32(uint32_t a, uint32_t b, float_status *s)
3360	{
3361	return float64_add(float32_to_float64(a, s),
3362	float32_to_float64(b, s), s);
3363
3364	}
3365
3366	RVVCALL(OPFVV2, vfwadd_vv_h_bf16, WOP_UUU_H, H4, H2, H2, vfwadd16_bf16)
3367	RVVCALL(OPFVV2, vfwadd_vv_h, WOP_UUU_H, H4, H2, H2, vfwadd16)
3368	RVVCALL(OPFVV2, vfwadd_vv_w, WOP_UUU_W, H8, H4, H4, vfwadd32)
3369	GEN_VEXT_VV_ENV(vfwadd_vv_h_bf16, 4)
3370	GEN_VEXT_VV_ENV(vfwadd_vv_h, 4)
3371	GEN_VEXT_VV_ENV(vfwadd_vv_w, 8)
3372	RVVCALL(OPFVF2, vfwadd_vf_h_bf16, WOP_UUU_H, H4, H2, vfwadd16_bf16)
3373	RVVCALL(OPFVF2, vfwadd_vf_h, WOP_UUU_H, H4, H2, vfwadd16)
3374	RVVCALL(OPFVF2, vfwadd_vf_w, WOP_UUU_W, H8, H4, vfwadd32)
3375	GEN_VEXT_VF(vfwadd_vf_h_bf16, 4)
3376	GEN_VEXT_VF(vfwadd_vf_h, 4)
3377	GEN_VEXT_VF(vfwadd_vf_w, 8)
3378
3379	static uint32_t vfwsub16_bf16(uint16_t a, uint16_t b, float_status *s)
3380	{
3381	return float32_sub(bfloat16_to_float32(a, s),
3382	bfloat16_to_float32(b, s), s);
3383	}
3384
3385	static uint32_t vfwsub16(uint16_t a, uint16_t b, float_status *s)
3386	{
3387	return float32_sub(float16_to_float32(a, true, s),
3388	float16_to_float32(b, true, s), s);
3389	}
3390
3391	static uint64_t vfwsub32(uint32_t a, uint32_t b, float_status *s)
3392	{
3393	return float64_sub(float32_to_float64(a, s),
3394	float32_to_float64(b, s), s);
3395
3396	}
3397
3398	RVVCALL(OPFVV2, vfwsub_vv_h_bf16, WOP_UUU_H, H4, H2, H2, vfwsub16_bf16)
3399	RVVCALL(OPFVV2, vfwsub_vv_h, WOP_UUU_H, H4, H2, H2, vfwsub16)
3400	RVVCALL(OPFVV2, vfwsub_vv_w, WOP_UUU_W, H8, H4, H4, vfwsub32)
3401	GEN_VEXT_VV_ENV(vfwsub_vv_h_bf16, 4)
3402	GEN_VEXT_VV_ENV(vfwsub_vv_h, 4)
3403	GEN_VEXT_VV_ENV(vfwsub_vv_w, 8)
3404	RVVCALL(OPFVF2, vfwsub_vf_h_bf16, WOP_UUU_H, H4, H2, vfwsub16_bf16)
3405	RVVCALL(OPFVF2, vfwsub_vf_h, WOP_UUU_H, H4, H2, vfwsub16)
3406	RVVCALL(OPFVF2, vfwsub_vf_w, WOP_UUU_W, H8, H4, vfwsub32)
3407	GEN_VEXT_VF(vfwsub_vf_h_bf16, 4)
3408	GEN_VEXT_VF(vfwsub_vf_h, 4)
3409	GEN_VEXT_VF(vfwsub_vf_w, 8)
3410
3411	static uint32_t vfwaddw16_bf16(uint32_t a, uint16_t b, float_status *s)
3412	{
3413	return float32_add(a, bfloat16_to_float32(b, s), s);
3414	}
3415
3416	static uint32_t vfwaddw16(uint32_t a, uint16_t b, float_status *s)
3417	{
3418	return float32_add(a, float16_to_float32(b, true, s), s);
3419	}
3420
3421	static uint64_t vfwaddw32(uint64_t a, uint32_t b, float_status *s)
3422	{
3423	return float64_add(a, float32_to_float64(b, s), s);
3424	}
3425
3426	RVVCALL(OPFVV2, vfwadd_wv_h_bf16, WOP_WUUU_H, H4, H2, H2, vfwaddw16_bf16)
3427	RVVCALL(OPFVV2, vfwadd_wv_h, WOP_WUUU_H, H4, H2, H2, vfwaddw16)
3428	RVVCALL(OPFVV2, vfwadd_wv_w, WOP_WUUU_W, H8, H4, H4, vfwaddw32)
3429	GEN_VEXT_VV_ENV(vfwadd_wv_h_bf16, 4)
3430	GEN_VEXT_VV_ENV(vfwadd_wv_h, 4)
3431	GEN_VEXT_VV_ENV(vfwadd_wv_w, 8)
3432	RVVCALL(OPFVF2, vfwadd_wf_h_bf16, WOP_WUUU_H, H4, H2, vfwaddw16_bf16)
3433	RVVCALL(OPFVF2, vfwadd_wf_h, WOP_WUUU_H, H4, H2, vfwaddw16)
3434	RVVCALL(OPFVF2, vfwadd_wf_w, WOP_WUUU_W, H8, H4, vfwaddw32)
3435	GEN_VEXT_VF(vfwadd_wf_h_bf16, 4)
3436	GEN_VEXT_VF(vfwadd_wf_h, 4)
3437	GEN_VEXT_VF(vfwadd_wf_w, 8)
3438
3439	static uint32_t vfwsubw16_bf16(uint32_t a, uint16_t b, float_status *s)
3440	{
3441	return float32_sub(a, bfloat16_to_float32(b, s), s);
3442	}
3443
3444	static uint32_t vfwsubw16(uint32_t a, uint16_t b, float_status *s)
3445	{
3446	return float32_sub(a, float16_to_float32(b, true, s), s);
3447	}
3448
3449	static uint64_t vfwsubw32(uint64_t a, uint32_t b, float_status *s)
3450	{
3451	return float64_sub(a, float32_to_float64(b, s), s);
3452	}
3453
3454	RVVCALL(OPFVV2, vfwsub_wv_h_bf16, WOP_WUUU_H, H4, H2, H2, vfwsubw16_bf16)
3455	RVVCALL(OPFVV2, vfwsub_wv_h, WOP_WUUU_H, H4, H2, H2, vfwsubw16)
3456	RVVCALL(OPFVV2, vfwsub_wv_w, WOP_WUUU_W, H8, H4, H4, vfwsubw32)
3457	GEN_VEXT_VV_ENV(vfwsub_wv_h_bf16, 4)
3458	GEN_VEXT_VV_ENV(vfwsub_wv_h, 4)
3459	GEN_VEXT_VV_ENV(vfwsub_wv_w, 8)
3460	RVVCALL(OPFVF2, vfwsub_wf_h_bf16, WOP_WUUU_H, H4, H2, vfwsubw16_bf16)
3461	RVVCALL(OPFVF2, vfwsub_wf_h, WOP_WUUU_H, H4, H2, vfwsubw16)
3462	RVVCALL(OPFVF2, vfwsub_wf_w, WOP_WUUU_W, H8, H4, vfwsubw32)
3463	GEN_VEXT_VF(vfwsub_wf_h_bf16, 4)
3464	GEN_VEXT_VF(vfwsub_wf_h, 4)
3465	GEN_VEXT_VF(vfwsub_wf_w, 8)
3466
3467	/* Vector Single-Width Floating-Point Multiply/Divide Instructions */
3468	RVVCALL(OPFVV2, vfmul_vv_h_bf16, OP_UUU_H, H2, H2, H2, bfloat16_mul)
3469	RVVCALL(OPFVV2, vfmul_vv_h, OP_UUU_H, H2, H2, H2, float16_mul)
3470	RVVCALL(OPFVV2, vfmul_vv_w, OP_UUU_W, H4, H4, H4, float32_mul)
3471	RVVCALL(OPFVV2, vfmul_vv_d, OP_UUU_D, H8, H8, H8, float64_mul)
3472	GEN_VEXT_VV_ENV(vfmul_vv_h_bf16, 2)
3473	GEN_VEXT_VV_ENV(vfmul_vv_h, 2)
3474	GEN_VEXT_VV_ENV(vfmul_vv_w, 4)
3475	GEN_VEXT_VV_ENV(vfmul_vv_d, 8)
3476	RVVCALL(OPFVF2, vfmul_vf_h_bf16, OP_UUU_H, H2, H2, bfloat16_mul)
3477	RVVCALL(OPFVF2, vfmul_vf_h, OP_UUU_H, H2, H2, float16_mul)
3478	RVVCALL(OPFVF2, vfmul_vf_w, OP_UUU_W, H4, H4, float32_mul)
3479	RVVCALL(OPFVF2, vfmul_vf_d, OP_UUU_D, H8, H8, float64_mul)
3480	GEN_VEXT_VF(vfmul_vf_h_bf16, 2)
3481	GEN_VEXT_VF(vfmul_vf_h, 2)
3482	GEN_VEXT_VF(vfmul_vf_w, 4)
3483	GEN_VEXT_VF(vfmul_vf_d, 8)
3484
3485	RVVCALL(OPFVV2, vfdiv_vv_h, OP_UUU_H, H2, H2, H2, float16_div)
3486	RVVCALL(OPFVV2, vfdiv_vv_w, OP_UUU_W, H4, H4, H4, float32_div)
3487	RVVCALL(OPFVV2, vfdiv_vv_d, OP_UUU_D, H8, H8, H8, float64_div)
3488	GEN_VEXT_VV_ENV(vfdiv_vv_h, 2)
3489	GEN_VEXT_VV_ENV(vfdiv_vv_w, 4)
3490	GEN_VEXT_VV_ENV(vfdiv_vv_d, 8)
3491	RVVCALL(OPFVF2, vfdiv_vf_h, OP_UUU_H, H2, H2, float16_div)
3492	RVVCALL(OPFVF2, vfdiv_vf_w, OP_UUU_W, H4, H4, float32_div)
3493	RVVCALL(OPFVF2, vfdiv_vf_d, OP_UUU_D, H8, H8, float64_div)
3494	GEN_VEXT_VF(vfdiv_vf_h, 2)
3495	GEN_VEXT_VF(vfdiv_vf_w, 4)
3496	GEN_VEXT_VF(vfdiv_vf_d, 8)
3497
3498	static uint16_t float16_rdiv(uint16_t a, uint16_t b, float_status *s)
3499	{
3500	return float16_div(b, a, s);
3501	}
3502
3503	static uint32_t float32_rdiv(uint32_t a, uint32_t b, float_status *s)
3504	{
3505	return float32_div(b, a, s);
3506	}
3507
3508	static uint64_t float64_rdiv(uint64_t a, uint64_t b, float_status *s)
3509	{
3510	return float64_div(b, a, s);
3511	}
3512
3513	RVVCALL(OPFVF2, vfrdiv_vf_h, OP_UUU_H, H2, H2, float16_rdiv)
3514	RVVCALL(OPFVF2, vfrdiv_vf_w, OP_UUU_W, H4, H4, float32_rdiv)
3515	RVVCALL(OPFVF2, vfrdiv_vf_d, OP_UUU_D, H8, H8, float64_rdiv)
3516	GEN_VEXT_VF(vfrdiv_vf_h, 2)
3517	GEN_VEXT_VF(vfrdiv_vf_w, 4)
3518	GEN_VEXT_VF(vfrdiv_vf_d, 8)
3519
3520	/* Vector Widening Floating-Point Multiply */
3521	static uint32_t vfwmul16_bf16(uint16_t a, uint16_t b, float_status *s)
3522	{
3523	return float32_mul(bfloat16_to_float32(a, s),
3524	bfloat16_to_float32(b, s), s);
3525	}
3526
3527	static uint32_t vfwmul16(uint16_t a, uint16_t b, float_status *s)
3528	{
3529	return float32_mul(float16_to_float32(a, true, s),
3530	float16_to_float32(b, true, s), s);
3531	}
3532
3533	static uint64_t vfwmul32(uint32_t a, uint32_t b, float_status *s)
3534	{
3535	return float64_mul(float32_to_float64(a, s),
3536	float32_to_float64(b, s), s);
3537
3538	}
3539
3540	RVVCALL(OPFVV2, vfwmul_vv_h_bf16, WOP_UUU_H, H4, H2, H2, vfwmul16_bf16)
3541	RVVCALL(OPFVV2, vfwmul_vv_h, WOP_UUU_H, H4, H2, H2, vfwmul16)
3542	RVVCALL(OPFVV2, vfwmul_vv_w, WOP_UUU_W, H8, H4, H4, vfwmul32)
3543	GEN_VEXT_VV_ENV(vfwmul_vv_h_bf16, 4)
3544	GEN_VEXT_VV_ENV(vfwmul_vv_h, 4)
3545	GEN_VEXT_VV_ENV(vfwmul_vv_w, 8)
3546	RVVCALL(OPFVF2, vfwmul_vf_h_bf16, WOP_UUU_H, H4, H2, vfwmul16_bf16)
3547	RVVCALL(OPFVF2, vfwmul_vf_h, WOP_UUU_H, H4, H2, vfwmul16)
3548	RVVCALL(OPFVF2, vfwmul_vf_w, WOP_UUU_W, H8, H4, vfwmul32)
3549	GEN_VEXT_VF(vfwmul_vf_h_bf16, 4)
3550	GEN_VEXT_VF(vfwmul_vf_h, 4)
3551	GEN_VEXT_VF(vfwmul_vf_w, 8)
3552
3553	/* Vector Single-Width Floating-Point Fused Multiply-Add Instructions */
3554	#define OPFVV3(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
3555	static void do_##NAME(void vd, void vs1, void *vs2, int i, \
3556	CPURISCVState *env) \
3557	{ \
3558	TX1 s1 = ((T1 )vs1 + HS1(i)); \
3559	TX2 s2 = ((T2 )vs2 + HS2(i)); \
3560	TD d = ((TD )vd + HD(i)); \
3561	((TD )vd + HD(i)) = OP(s2, s1, d, &env->fp_status); \
3562	}
3563
3564	static uint16_t fmacc16_bf16(uint16_t a, uint16_t b, uint16_t d,
3565	float_status *s)
3566	{
3567	return bfloat16_muladd(a, b, d, 0, s);
3568	}
3569
3570	static uint16_t fmacc16(uint16_t a, uint16_t b, uint16_t d, float_status *s)
3571	{
3572	return float16_muladd(a, b, d, 0, s);
3573	}
3574
3575	static uint32_t fmacc32(uint32_t a, uint32_t b, uint32_t d, float_status *s)
3576	{
3577	return float32_muladd(a, b, d, 0, s);
3578	}
3579
3580	static uint64_t fmacc64(uint64_t a, uint64_t b, uint64_t d, float_status *s)
3581	{
3582	return float64_muladd(a, b, d, 0, s);
3583	}
3584
3585	RVVCALL(OPFVV3, vfmacc_vv_h_bf16, OP_UUU_H, H2, H2, H2, fmacc16_bf16)
3586	RVVCALL(OPFVV3, vfmacc_vv_h, OP_UUU_H, H2, H2, H2, fmacc16)
3587	RVVCALL(OPFVV3, vfmacc_vv_w, OP_UUU_W, H4, H4, H4, fmacc32)
3588	RVVCALL(OPFVV3, vfmacc_vv_d, OP_UUU_D, H8, H8, H8, fmacc64)
3589	GEN_VEXT_VV_ENV(vfmacc_vv_h_bf16, 2)
3590	GEN_VEXT_VV_ENV(vfmacc_vv_h, 2)
3591	GEN_VEXT_VV_ENV(vfmacc_vv_w, 4)
3592	GEN_VEXT_VV_ENV(vfmacc_vv_d, 8)
3593
3594	#define OPFVF3(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \
3595	static void do_##NAME(void vd, uint64_t s1, void vs2, int i, \
3596	CPURISCVState *env) \
3597	{ \
3598	TX2 s2 = ((T2 )vs2 + HS2(i)); \
3599	TD d = ((TD )vd + HD(i)); \
3600	((TD )vd + HD(i)) = OP(s2, (TX1)(T1)s1, d, &env->fp_status);\
3601	}
3602
3603	RVVCALL(OPFVF3, vfmacc_vf_h_bf16, OP_UUU_H, H2, H2, fmacc16_bf16)
3604	RVVCALL(OPFVF3, vfmacc_vf_h, OP_UUU_H, H2, H2, fmacc16)
3605	RVVCALL(OPFVF3, vfmacc_vf_w, OP_UUU_W, H4, H4, fmacc32)
3606	RVVCALL(OPFVF3, vfmacc_vf_d, OP_UUU_D, H8, H8, fmacc64)
3607	GEN_VEXT_VF(vfmacc_vf_h_bf16, 2)
3608	GEN_VEXT_VF(vfmacc_vf_h, 2)
3609	GEN_VEXT_VF(vfmacc_vf_w, 4)
3610	GEN_VEXT_VF(vfmacc_vf_d, 8)
3611
3612	static uint16_t fnmacc16_bf16(uint16_t a, uint16_t b, uint16_t d,
3613	float_status *s)
3614	{
3615	return bfloat16_muladd(a, b, d, float_muladd_negate_c \|
3616	float_muladd_negate_product, s);
3617	}
3618
3619	static uint16_t fnmacc16(uint16_t a, uint16_t b, uint16_t d, float_status *s)
3620	{
3621	return float16_muladd(a, b, d, float_muladd_negate_c \|
3622	float_muladd_negate_product, s);
3623	}
3624
3625	static uint32_t fnmacc32(uint32_t a, uint32_t b, uint32_t d, float_status *s)
3626	{
3627	return float32_muladd(a, b, d, float_muladd_negate_c \|
3628	float_muladd_negate_product, s);
3629	}
3630
3631	static uint64_t fnmacc64(uint64_t a, uint64_t b, uint64_t d, float_status *s)
3632	{
3633	return float64_muladd(a, b, d, float_muladd_negate_c \|
3634	float_muladd_negate_product, s);
3635	}
3636
3637	RVVCALL(OPFVV3, vfnmacc_vv_h_bf16, OP_UUU_H, H2, H2, H2, fnmacc16_bf16)
3638	RVVCALL(OPFVV3, vfnmacc_vv_h, OP_UUU_H, H2, H2, H2, fnmacc16)
3639	RVVCALL(OPFVV3, vfnmacc_vv_w, OP_UUU_W, H4, H4, H4, fnmacc32)
3640	RVVCALL(OPFVV3, vfnmacc_vv_d, OP_UUU_D, H8, H8, H8, fnmacc64)
3641	GEN_VEXT_VV_ENV(vfnmacc_vv_h_bf16, 2)
3642	GEN_VEXT_VV_ENV(vfnmacc_vv_h, 2)
3643	GEN_VEXT_VV_ENV(vfnmacc_vv_w, 4)
3644	GEN_VEXT_VV_ENV(vfnmacc_vv_d, 8)
3645	RVVCALL(OPFVF3, vfnmacc_vf_h_bf16, OP_UUU_H, H2, H2, fnmacc16_bf16)
3646	RVVCALL(OPFVF3, vfnmacc_vf_h, OP_UUU_H, H2, H2, fnmacc16)
3647	RVVCALL(OPFVF3, vfnmacc_vf_w, OP_UUU_W, H4, H4, fnmacc32)
3648	RVVCALL(OPFVF3, vfnmacc_vf_d, OP_UUU_D, H8, H8, fnmacc64)
3649	GEN_VEXT_VF(vfnmacc_vf_h_bf16, 2)
3650	GEN_VEXT_VF(vfnmacc_vf_h, 2)
3651	GEN_VEXT_VF(vfnmacc_vf_w, 4)
3652	GEN_VEXT_VF(vfnmacc_vf_d, 8)
3653
3654	static uint16_t fmsac16_bf16(uint16_t a, uint16_t b, uint16_t d,
3655	float_status *s)
3656	{
3657	return bfloat16_muladd(a, b, d, float_muladd_negate_c, s);
3658	}
3659
3660	static uint16_t fmsac16(uint16_t a, uint16_t b, uint16_t d, float_status *s)
3661	{
3662	return float16_muladd(a, b, d, float_muladd_negate_c, s);
3663	}
3664
3665	static uint32_t fmsac32(uint32_t a, uint32_t b, uint32_t d, float_status *s)
3666	{
3667	return float32_muladd(a, b, d, float_muladd_negate_c, s);
3668	}
3669
3670	static uint64_t fmsac64(uint64_t a, uint64_t b, uint64_t d, float_status *s)
3671	{
3672	return float64_muladd(a, b, d, float_muladd_negate_c, s);
3673	}
3674
3675	RVVCALL(OPFVV3, vfmsac_vv_h_bf16, OP_UUU_H, H2, H2, H2, fmsac16_bf16)
3676	RVVCALL(OPFVV3, vfmsac_vv_h, OP_UUU_H, H2, H2, H2, fmsac16)
3677	RVVCALL(OPFVV3, vfmsac_vv_w, OP_UUU_W, H4, H4, H4, fmsac32)
3678	RVVCALL(OPFVV3, vfmsac_vv_d, OP_UUU_D, H8, H8, H8, fmsac64)
3679	GEN_VEXT_VV_ENV(vfmsac_vv_h_bf16, 2)
3680	GEN_VEXT_VV_ENV(vfmsac_vv_h, 2)
3681	GEN_VEXT_VV_ENV(vfmsac_vv_w, 4)
3682	GEN_VEXT_VV_ENV(vfmsac_vv_d, 8)
3683	RVVCALL(OPFVF3, vfmsac_vf_h_bf16, OP_UUU_H, H2, H2, fmsac16_bf16)
3684	RVVCALL(OPFVF3, vfmsac_vf_h, OP_UUU_H, H2, H2, fmsac16)
3685	RVVCALL(OPFVF3, vfmsac_vf_w, OP_UUU_W, H4, H4, fmsac32)
3686	RVVCALL(OPFVF3, vfmsac_vf_d, OP_UUU_D, H8, H8, fmsac64)
3687	GEN_VEXT_VF(vfmsac_vf_h_bf16, 2)
3688	GEN_VEXT_VF(vfmsac_vf_h, 2)
3689	GEN_VEXT_VF(vfmsac_vf_w, 4)
3690	GEN_VEXT_VF(vfmsac_vf_d, 8)
3691
3692	static uint16_t fnmsac16_bf16(uint16_t a, uint16_t b, uint16_t d,
3693	float_status *s)
3694	{
3695	return bfloat16_muladd(a, b, d, float_muladd_negate_product, s);
3696	}
3697
3698	static uint16_t fnmsac16(uint16_t a, uint16_t b, uint16_t d, float_status *s)
3699	{
3700	return float16_muladd(a, b, d, float_muladd_negate_product, s);
3701	}
3702
3703	static uint32_t fnmsac32(uint32_t a, uint32_t b, uint32_t d, float_status *s)
3704	{
3705	return float32_muladd(a, b, d, float_muladd_negate_product, s);
3706	}
3707
3708	static uint64_t fnmsac64(uint64_t a, uint64_t b, uint64_t d, float_status *s)
3709	{
3710	return float64_muladd(a, b, d, float_muladd_negate_product, s);
3711	}
3712
3713	RVVCALL(OPFVV3, vfnmsac_vv_h_bf16, OP_UUU_H, H2, H2, H2, fnmsac16_bf16)
3714	RVVCALL(OPFVV3, vfnmsac_vv_h, OP_UUU_H, H2, H2, H2, fnmsac16)
3715	RVVCALL(OPFVV3, vfnmsac_vv_w, OP_UUU_W, H4, H4, H4, fnmsac32)
3716	RVVCALL(OPFVV3, vfnmsac_vv_d, OP_UUU_D, H8, H8, H8, fnmsac64)
3717	GEN_VEXT_VV_ENV(vfnmsac_vv_h_bf16, 2)
3718	GEN_VEXT_VV_ENV(vfnmsac_vv_h, 2)
3719	GEN_VEXT_VV_ENV(vfnmsac_vv_w, 4)
3720	GEN_VEXT_VV_ENV(vfnmsac_vv_d, 8)
3721	RVVCALL(OPFVF3, vfnmsac_vf_h_bf16, OP_UUU_H, H2, H2, fnmsac16_bf16)
3722	RVVCALL(OPFVF3, vfnmsac_vf_h, OP_UUU_H, H2, H2, fnmsac16)
3723	RVVCALL(OPFVF3, vfnmsac_vf_w, OP_UUU_W, H4, H4, fnmsac32)
3724	RVVCALL(OPFVF3, vfnmsac_vf_d, OP_UUU_D, H8, H8, fnmsac64)
3725	GEN_VEXT_VF(vfnmsac_vf_h_bf16, 2)
3726	GEN_VEXT_VF(vfnmsac_vf_h, 2)
3727	GEN_VEXT_VF(vfnmsac_vf_w, 4)
3728	GEN_VEXT_VF(vfnmsac_vf_d, 8)
3729
3730	static uint16_t fmadd16_bf16(uint16_t a, uint16_t b, uint16_t d,
3731	float_status *s)
3732	{
3733	return bfloat16_muladd(d, b, a, 0, s);
3734	}
3735
3736	static uint16_t fmadd16(uint16_t a, uint16_t b, uint16_t d, float_status *s)
3737	{
3738	return float16_muladd(d, b, a, 0, s);
3739	}
3740
3741	static uint32_t fmadd32(uint32_t a, uint32_t b, uint32_t d, float_status *s)
3742	{
3743	return float32_muladd(d, b, a, 0, s);
3744	}
3745
3746	static uint64_t fmadd64(uint64_t a, uint64_t b, uint64_t d, float_status *s)
3747	{
3748	return float64_muladd(d, b, a, 0, s);
3749	}
3750
3751	RVVCALL(OPFVV3, vfmadd_vv_h_bf16, OP_UUU_H, H2, H2, H2, fmadd16_bf16)
3752	RVVCALL(OPFVV3, vfmadd_vv_h, OP_UUU_H, H2, H2, H2, fmadd16)
3753	RVVCALL(OPFVV3, vfmadd_vv_w, OP_UUU_W, H4, H4, H4, fmadd32)
3754	RVVCALL(OPFVV3, vfmadd_vv_d, OP_UUU_D, H8, H8, H8, fmadd64)
3755	GEN_VEXT_VV_ENV(vfmadd_vv_h_bf16, 2)
3756	GEN_VEXT_VV_ENV(vfmadd_vv_h, 2)
3757	GEN_VEXT_VV_ENV(vfmadd_vv_w, 4)
3758	GEN_VEXT_VV_ENV(vfmadd_vv_d, 8)
3759	RVVCALL(OPFVF3, vfmadd_vf_h_bf16, OP_UUU_H, H2, H2, fmadd16_bf16)
3760	RVVCALL(OPFVF3, vfmadd_vf_h, OP_UUU_H, H2, H2, fmadd16)
3761	RVVCALL(OPFVF3, vfmadd_vf_w, OP_UUU_W, H4, H4, fmadd32)
3762	RVVCALL(OPFVF3, vfmadd_vf_d, OP_UUU_D, H8, H8, fmadd64)
3763	GEN_VEXT_VF(vfmadd_vf_h_bf16, 2)
3764	GEN_VEXT_VF(vfmadd_vf_h, 2)
3765	GEN_VEXT_VF(vfmadd_vf_w, 4)
3766	GEN_VEXT_VF(vfmadd_vf_d, 8)
3767
3768	static uint16_t fnmadd16_bf16(uint16_t a, uint16_t b, uint16_t d,
3769	float_status *s)
3770	{
3771	return bfloat16_muladd(d, b, a, float_muladd_negate_c \|
3772	float_muladd_negate_product, s);
3773	}
3774
3775	static uint16_t fnmadd16(uint16_t a, uint16_t b, uint16_t d, float_status *s)
3776	{
3777	return float16_muladd(d, b, a, float_muladd_negate_c \|
3778	float_muladd_negate_product, s);
3779	}
3780
3781	static uint32_t fnmadd32(uint32_t a, uint32_t b, uint32_t d, float_status *s)
3782	{
3783	return float32_muladd(d, b, a, float_muladd_negate_c \|
3784	float_muladd_negate_product, s);
3785	}
3786
3787	static uint64_t fnmadd64(uint64_t a, uint64_t b, uint64_t d, float_status *s)
3788	{
3789	return float64_muladd(d, b, a, float_muladd_negate_c \|
3790	float_muladd_negate_product, s);
3791	}
3792
3793	RVVCALL(OPFVV3, vfnmadd_vv_h_bf16, OP_UUU_H, H2, H2, H2, fnmadd16_bf16)
3794	RVVCALL(OPFVV3, vfnmadd_vv_h, OP_UUU_H, H2, H2, H2, fnmadd16)
3795	RVVCALL(OPFVV3, vfnmadd_vv_w, OP_UUU_W, H4, H4, H4, fnmadd32)
3796	RVVCALL(OPFVV3, vfnmadd_vv_d, OP_UUU_D, H8, H8, H8, fnmadd64)
3797	GEN_VEXT_VV_ENV(vfnmadd_vv_h_bf16, 2)
3798	GEN_VEXT_VV_ENV(vfnmadd_vv_h, 2)
3799	GEN_VEXT_VV_ENV(vfnmadd_vv_w, 4)
3800	GEN_VEXT_VV_ENV(vfnmadd_vv_d, 8)
3801	RVVCALL(OPFVF3, vfnmadd_vf_h_bf16, OP_UUU_H, H2, H2, fnmadd16_bf16)
3802	RVVCALL(OPFVF3, vfnmadd_vf_h, OP_UUU_H, H2, H2, fnmadd16)
3803	RVVCALL(OPFVF3, vfnmadd_vf_w, OP_UUU_W, H4, H4, fnmadd32)
3804	RVVCALL(OPFVF3, vfnmadd_vf_d, OP_UUU_D, H8, H8, fnmadd64)
3805	GEN_VEXT_VF(vfnmadd_vf_h_bf16, 2)
3806	GEN_VEXT_VF(vfnmadd_vf_h, 2)
3807	GEN_VEXT_VF(vfnmadd_vf_w, 4)
3808	GEN_VEXT_VF(vfnmadd_vf_d, 8)
3809
3810	static uint16_t fmsub16_bf16(uint16_t a, uint16_t b, uint16_t d,
3811	float_status *s)
3812	{
3813	return bfloat16_muladd(d, b, a, float_muladd_negate_c, s);
3814	}
3815
3816	static uint16_t fmsub16(uint16_t a, uint16_t b, uint16_t d, float_status *s)
3817	{
3818	return float16_muladd(d, b, a, float_muladd_negate_c, s);
3819	}
3820
3821	static uint32_t fmsub32(uint32_t a, uint32_t b, uint32_t d, float_status *s)
3822	{
3823	return float32_muladd(d, b, a, float_muladd_negate_c, s);
3824	}
3825
3826	static uint64_t fmsub64(uint64_t a, uint64_t b, uint64_t d, float_status *s)
3827	{
3828	return float64_muladd(d, b, a, float_muladd_negate_c, s);
3829	}
3830
3831	RVVCALL(OPFVV3, vfmsub_vv_h_bf16, OP_UUU_H, H2, H2, H2, fmsub16_bf16)
3832	RVVCALL(OPFVV3, vfmsub_vv_h, OP_UUU_H, H2, H2, H2, fmsub16)
3833	RVVCALL(OPFVV3, vfmsub_vv_w, OP_UUU_W, H4, H4, H4, fmsub32)
3834	RVVCALL(OPFVV3, vfmsub_vv_d, OP_UUU_D, H8, H8, H8, fmsub64)
3835	GEN_VEXT_VV_ENV(vfmsub_vv_h_bf16, 2)
3836	GEN_VEXT_VV_ENV(vfmsub_vv_h, 2)
3837	GEN_VEXT_VV_ENV(vfmsub_vv_w, 4)
3838	GEN_VEXT_VV_ENV(vfmsub_vv_d, 8)
3839	RVVCALL(OPFVF3, vfmsub_vf_h_bf16, OP_UUU_H, H2, H2, fmsub16_bf16)
3840	RVVCALL(OPFVF3, vfmsub_vf_h, OP_UUU_H, H2, H2, fmsub16)
3841	RVVCALL(OPFVF3, vfmsub_vf_w, OP_UUU_W, H4, H4, fmsub32)
3842	RVVCALL(OPFVF3, vfmsub_vf_d, OP_UUU_D, H8, H8, fmsub64)
3843	GEN_VEXT_VF(vfmsub_vf_h_bf16, 2)
3844	GEN_VEXT_VF(vfmsub_vf_h, 2)
3845	GEN_VEXT_VF(vfmsub_vf_w, 4)
3846	GEN_VEXT_VF(vfmsub_vf_d, 8)
3847
3848	static uint16_t fnmsub16_bf16(uint16_t a, uint16_t b, uint16_t d,
3849	float_status *s)
3850	{
3851	return bfloat16_muladd(d, b, a, float_muladd_negate_product, s);
3852	}
3853
3854	static uint16_t fnmsub16(uint16_t a, uint16_t b, uint16_t d, float_status *s)
3855	{
3856	return float16_muladd(d, b, a, float_muladd_negate_product, s);
3857	}
3858
3859	static uint32_t fnmsub32(uint32_t a, uint32_t b, uint32_t d, float_status *s)
3860	{
3861	return float32_muladd(d, b, a, float_muladd_negate_product, s);
3862	}
3863
3864	static uint64_t fnmsub64(uint64_t a, uint64_t b, uint64_t d, float_status *s)
3865	{
3866	return float64_muladd(d, b, a, float_muladd_negate_product, s);
3867	}
3868
3869	RVVCALL(OPFVV3, vfnmsub_vv_h_bf16, OP_UUU_H, H2, H2, H2, fnmsub16_bf16)
3870	RVVCALL(OPFVV3, vfnmsub_vv_h, OP_UUU_H, H2, H2, H2, fnmsub16)
3871	RVVCALL(OPFVV3, vfnmsub_vv_w, OP_UUU_W, H4, H4, H4, fnmsub32)
3872	RVVCALL(OPFVV3, vfnmsub_vv_d, OP_UUU_D, H8, H8, H8, fnmsub64)
3873	GEN_VEXT_VV_ENV(vfnmsub_vv_h_bf16, 2)
3874	GEN_VEXT_VV_ENV(vfnmsub_vv_h, 2)
3875	GEN_VEXT_VV_ENV(vfnmsub_vv_w, 4)
3876	GEN_VEXT_VV_ENV(vfnmsub_vv_d, 8)
3877	RVVCALL(OPFVF3, vfnmsub_vf_h_bf16, OP_UUU_H, H2, H2, fnmsub16_bf16)
3878	RVVCALL(OPFVF3, vfnmsub_vf_h, OP_UUU_H, H2, H2, fnmsub16)
3879	RVVCALL(OPFVF3, vfnmsub_vf_w, OP_UUU_W, H4, H4, fnmsub32)
3880	RVVCALL(OPFVF3, vfnmsub_vf_d, OP_UUU_D, H8, H8, fnmsub64)
3881	GEN_VEXT_VF(vfnmsub_vf_h_bf16, 2)
3882	GEN_VEXT_VF(vfnmsub_vf_h, 2)
3883	GEN_VEXT_VF(vfnmsub_vf_w, 4)
3884	GEN_VEXT_VF(vfnmsub_vf_d, 8)
3885
3886	/* Vector Widening Floating-Point Fused Multiply-Add Instructions */
3887	static uint32_t fwmacc16(uint16_t a, uint16_t b, uint32_t d, float_status *s)
3888	{
3889	return float32_muladd(float16_to_float32(a, true, s),
3890	float16_to_float32(b, true, s), d, 0, s);
3891	}
3892
3893	static uint64_t fwmacc32(uint32_t a, uint32_t b, uint64_t d, float_status *s)
3894	{
3895	return float64_muladd(float32_to_float64(a, s),
3896	float32_to_float64(b, s), d, 0, s);
3897	}
3898
3899	RVVCALL(OPFVV3, vfwmacc_vv_h, WOP_UUU_H, H4, H2, H2, fwmacc16)
3900	RVVCALL(OPFVV3, vfwmacc_vv_w, WOP_UUU_W, H8, H4, H4, fwmacc32)
3901	GEN_VEXT_VV_ENV(vfwmacc_vv_h, 4)
3902	GEN_VEXT_VV_ENV(vfwmacc_vv_w, 8)
3903	RVVCALL(OPFVF3, vfwmacc_vf_h, WOP_UUU_H, H4, H2, fwmacc16)
3904	RVVCALL(OPFVF3, vfwmacc_vf_w, WOP_UUU_W, H8, H4, fwmacc32)
3905	GEN_VEXT_VF(vfwmacc_vf_h, 4)
3906	GEN_VEXT_VF(vfwmacc_vf_w, 8)
3907
3908	static uint32_t fwmaccbf16(uint16_t a, uint16_t b, uint32_t d, float_status *s)
3909	{
3910	return float32_muladd(bfloat16_to_float32(a, s),
3911	bfloat16_to_float32(b, s), d, 0, s);
3912	}
3913
3914	RVVCALL(OPFVV3, vfwmaccbf16_vv, WOP_UUU_H, H4, H2, H2, fwmaccbf16)
3915	GEN_VEXT_VV_ENV(vfwmaccbf16_vv, 4)
3916	RVVCALL(OPFVF3, vfwmaccbf16_vf, WOP_UUU_H, H4, H2, fwmaccbf16)
3917	GEN_VEXT_VF(vfwmaccbf16_vf, 4)
3918
3919	static uint32_t fwnmacc16_bf16(uint16_t a, uint16_t b, uint32_t d,
3920	float_status *s)
3921	{
3922	return float32_muladd(bfloat16_to_float32(a, s),
3923	bfloat16_to_float32(b, s), d,
3924	float_muladd_negate_c \| float_muladd_negate_product,
3925	s);
3926	}
3927
3928	static uint32_t fwnmacc16(uint16_t a, uint16_t b, uint32_t d, float_status *s)
3929	{
3930	return float32_muladd(float16_to_float32(a, true, s),
3931	float16_to_float32(b, true, s), d,
3932	float_muladd_negate_c \| float_muladd_negate_product,
3933	s);
3934	}
3935
3936	static uint64_t fwnmacc32(uint32_t a, uint32_t b, uint64_t d, float_status *s)
3937	{
3938	return float64_muladd(float32_to_float64(a, s), float32_to_float64(b, s),
3939	d, float_muladd_negate_c \|
3940	float_muladd_negate_product, s);
3941	}
3942
3943	RVVCALL(OPFVV3, vfwnmacc_vv_h_bf16, WOP_UUU_H, H4, H2, H2, fwnmacc16_bf16)
3944	RVVCALL(OPFVV3, vfwnmacc_vv_h, WOP_UUU_H, H4, H2, H2, fwnmacc16)
3945	RVVCALL(OPFVV3, vfwnmacc_vv_w, WOP_UUU_W, H8, H4, H4, fwnmacc32)
3946	GEN_VEXT_VV_ENV(vfwnmacc_vv_h_bf16, 4)
3947	GEN_VEXT_VV_ENV(vfwnmacc_vv_h, 4)
3948	GEN_VEXT_VV_ENV(vfwnmacc_vv_w, 8)
3949	RVVCALL(OPFVF3, vfwnmacc_vf_h_bf16, WOP_UUU_H, H4, H2, fwnmacc16_bf16)
3950	RVVCALL(OPFVF3, vfwnmacc_vf_h, WOP_UUU_H, H4, H2, fwnmacc16)
3951	RVVCALL(OPFVF3, vfwnmacc_vf_w, WOP_UUU_W, H8, H4, fwnmacc32)
3952	GEN_VEXT_VF(vfwnmacc_vf_h_bf16, 4)
3953	GEN_VEXT_VF(vfwnmacc_vf_h, 4)
3954	GEN_VEXT_VF(vfwnmacc_vf_w, 8)
3955
3956	static uint32_t fwmsac16_bf16(uint16_t a, uint16_t b, uint32_t d,
3957	float_status *s)
3958	{
3959	return float32_muladd(bfloat16_to_float32(a, s),
3960	bfloat16_to_float32(b, s), d,
3961	float_muladd_negate_c, s);
3962	}
3963
3964	static uint32_t fwmsac16(uint16_t a, uint16_t b, uint32_t d, float_status *s)
3965	{
3966	return float32_muladd(float16_to_float32(a, true, s),
3967	float16_to_float32(b, true, s), d,
3968	float_muladd_negate_c, s);
3969	}
3970
3971	static uint64_t fwmsac32(uint32_t a, uint32_t b, uint64_t d, float_status *s)
3972	{
3973	return float64_muladd(float32_to_float64(a, s),
3974	float32_to_float64(b, s), d,
3975	float_muladd_negate_c, s);
3976	}
3977
3978	RVVCALL(OPFVV3, vfwmsac_vv_h_bf16, WOP_UUU_H, H4, H2, H2, fwmsac16_bf16)
3979	RVVCALL(OPFVV3, vfwmsac_vv_h, WOP_UUU_H, H4, H2, H2, fwmsac16)
3980	RVVCALL(OPFVV3, vfwmsac_vv_w, WOP_UUU_W, H8, H4, H4, fwmsac32)
3981	GEN_VEXT_VV_ENV(vfwmsac_vv_h_bf16, 4)
3982	GEN_VEXT_VV_ENV(vfwmsac_vv_h, 4)
3983	GEN_VEXT_VV_ENV(vfwmsac_vv_w, 8)
3984	RVVCALL(OPFVF3, vfwmsac_vf_h_bf16, WOP_UUU_H, H4, H2, fwmsac16_bf16)
3985	RVVCALL(OPFVF3, vfwmsac_vf_h, WOP_UUU_H, H4, H2, fwmsac16)
3986	RVVCALL(OPFVF3, vfwmsac_vf_w, WOP_UUU_W, H8, H4, fwmsac32)
3987	GEN_VEXT_VF(vfwmsac_vf_h_bf16, 4)
3988	GEN_VEXT_VF(vfwmsac_vf_h, 4)
3989	GEN_VEXT_VF(vfwmsac_vf_w, 8)
3990
3991	static uint32_t fwnmsac16_bf16(uint16_t a, uint16_t b, uint32_t d,
3992	float_status *s)
3993	{
3994	return float32_muladd(bfloat16_to_float32(a, s),
3995	bfloat16_to_float32(b, s), d,
3996	float_muladd_negate_product, s);
3997	}
3998
3999	static uint32_t fwnmsac16(uint16_t a, uint16_t b, uint32_t d, float_status *s)
4000	{
4001	return float32_muladd(float16_to_float32(a, true, s),
4002	float16_to_float32(b, true, s), d,
4003	float_muladd_negate_product, s);
4004	}
4005
4006	static uint64_t fwnmsac32(uint32_t a, uint32_t b, uint64_t d, float_status *s)
4007	{
4008	return float64_muladd(float32_to_float64(a, s),
4009	float32_to_float64(b, s), d,
4010	float_muladd_negate_product, s);
4011	}
4012
4013	RVVCALL(OPFVV3, vfwnmsac_vv_h_bf16, WOP_UUU_H, H4, H2, H2, fwnmsac16_bf16)
4014	RVVCALL(OPFVV3, vfwnmsac_vv_h, WOP_UUU_H, H4, H2, H2, fwnmsac16)
4015	RVVCALL(OPFVV3, vfwnmsac_vv_w, WOP_UUU_W, H8, H4, H4, fwnmsac32)
4016	GEN_VEXT_VV_ENV(vfwnmsac_vv_h_bf16, 4)
4017	GEN_VEXT_VV_ENV(vfwnmsac_vv_h, 4)
4018	GEN_VEXT_VV_ENV(vfwnmsac_vv_w, 8)
4019	RVVCALL(OPFVF3, vfwnmsac_vf_h_bf16, WOP_UUU_H, H4, H2, fwnmsac16_bf16)
4020	RVVCALL(OPFVF3, vfwnmsac_vf_h, WOP_UUU_H, H4, H2, fwnmsac16)
4021	RVVCALL(OPFVF3, vfwnmsac_vf_w, WOP_UUU_W, H8, H4, fwnmsac32)
4022	GEN_VEXT_VF(vfwnmsac_vf_h_bf16, 4)
4023	GEN_VEXT_VF(vfwnmsac_vf_h, 4)
4024	GEN_VEXT_VF(vfwnmsac_vf_w, 8)
4025
4026	/* Vector Floating-Point Square-Root Instruction */
4027	#define OPFVV1(NAME, TD, T2, TX2, HD, HS2, OP) \
4028	static void do_##NAME(void vd, void vs2, int i, \
4029	CPURISCVState *env) \
4030	{ \
4031	TX2 s2 = ((T2 )vs2 + HS2(i)); \
4032	((TD )vd + HD(i)) = OP(s2, &env->fp_status); \
4033	}
4034
4035	#define GEN_VEXT_V_ENV(NAME, ESZ) \
4036	void HELPER(NAME)(void vd, void v0, void *vs2, \
4037	CPURISCVState *env, uint32_t desc) \
4038	{ \
4039	uint32_t vm = vext_vm(desc); \
4040	uint32_t vl = env->vl; \
4041	uint32_t total_elems = \
4042	vext_get_total_elems(env, desc, ESZ); \
4043	uint32_t vta = vext_vta(desc); \
4044	uint32_t vma = vext_vma(desc); \
4045	uint32_t i; \
4046	FloatExceptionFlags pre_fflag = \
4047	get_float_exception_flags(&env->fp_status); \
4048	\
4049	VSTART_CHECK_EARLY_EXIT(env, vl); \
4050	\
4051	if (vl == 0) { \
4052	return; \
4053	} \
4054	for (i = env->vstart; i < vl; i++) { \
4055	if (!vm && !vext_elem_mask(v0, i)) { \
4056	/* set masked-off elements to 1s */ \
4057	vext_set_elems_1s(vd, vma, i * ESZ, \
4058	(i + 1) * ESZ); \
4059	continue; \
4060	} \
4061	do_##NAME(vd, vs2, i, env); \
4062	} \
4063	env->vstart = 0; \
4064	vext_set_elems_1s(vd, vta, vl * ESZ, \
4065	total_elems * ESZ); \
4066	riscv_cpu_check_fflags(env, pre_fflag); \
4067	}
4068
4069	RVVCALL(OPFVV1, vfsqrt_v_h, OP_UU_H, H2, H2, float16_sqrt)
4070	RVVCALL(OPFVV1, vfsqrt_v_w, OP_UU_W, H4, H4, float32_sqrt)
4071	RVVCALL(OPFVV1, vfsqrt_v_d, OP_UU_D, H8, H8, float64_sqrt)
4072	GEN_VEXT_V_ENV(vfsqrt_v_h, 2)
4073	GEN_VEXT_V_ENV(vfsqrt_v_w, 4)
4074	GEN_VEXT_V_ENV(vfsqrt_v_d, 8)
4075
4076	/*
4077	* Vector Floating-Point Reciprocal Square-Root Estimate Instruction
4078	*
4079	* Adapted from riscv-v-spec recip.c:
4080	* https://github.com/riscv/riscv-v-spec/blob/master/recip.c
4081	*/
4082	static uint64_t frsqrt7(uint64_t f, int exp_size, int frac_size)
4083	{
4084	uint64_t sign = extract64(f, frac_size + exp_size, 1);
4085	uint64_t exp = extract64(f, frac_size, exp_size);
4086	uint64_t frac = extract64(f, 0, frac_size);
4087
4088	const uint8_t lookup_table[] = {
4089	52, 51, 50, 48, 47, 46, 44, 43,
4090	42, 41, 40, 39, 38, 36, 35, 34,
4091	33, 32, 31, 30, 30, 29, 28, 27,
4092	26, 25, 24, 23, 23, 22, 21, 20,
4093	19, 19, 18, 17, 16, 16, 15, 14,
4094	14, 13, 12, 12, 11, 10, 10, 9,
4095	9, 8, 7, 7, 6, 6, 5, 4,
4096	4, 3, 3, 2, 2, 1, 1, 0,
4097	127, 125, 123, 121, 119, 118, 116, 114,
4098	113, 111, 109, 108, 106, 105, 103, 102,
4099	100, 99, 97, 96, 95, 93, 92, 91,
4100	90, 88, 87, 86, 85, 84, 83, 82,
4101	80, 79, 78, 77, 76, 75, 74, 73,
4102	72, 71, 70, 70, 69, 68, 67, 66,
4103	65, 64, 63, 63, 62, 61, 60, 59,
4104	59, 58, 57, 56, 56, 55, 54, 53
4105	};
4106	const int precision = 7;
4107
4108	if (exp == 0 && frac != 0) { /* subnormal */
4109	/* Normalize the subnormal. */
4110	while (extract64(frac, frac_size - 1, 1) == 0) {
4111	exp--;
4112	frac <<= 1;
4113	}
4114
4115	frac = (frac << 1) & MAKE_64BIT_MASK(0, frac_size);
4116	}
4117
4118	int idx = ((exp & 1) << (precision - 1)) \|
4119	(frac >> (frac_size - precision + 1));
4120	uint64_t out_frac = (uint64_t)(lookup_table[idx]) <<
4121	(frac_size - precision);
4122	uint64_t out_exp = (3 * MAKE_64BIT_MASK(0, exp_size - 1) + ~exp) / 2;
4123
4124	uint64_t val = 0;
4125	val = deposit64(val, 0, frac_size, out_frac);
4126	val = deposit64(val, frac_size, exp_size, out_exp);
4127	val = deposit64(val, frac_size + exp_size, 1, sign);
4128	return val;
4129	}
4130
4131	static bfloat16 frsqrt7_h_bf16(bfloat16 f, float_status *s)
4132	{
4133	int exp_size = 8, frac_size = 7;
4134	bool sign = bfloat16_is_neg(f);
4135
4136	/*
4137	* frsqrt7(sNaN) = canonical NaN
4138	* frsqrt7(-inf) = canonical NaN
4139	* frsqrt7(-normal) = canonical NaN
4140	* frsqrt7(-subnormal) = canonical NaN
4141	*/
4142	if (bfloat16_is_signaling_nan(f, s) \|\|
4143	(bfloat16_is_infinity(f) && sign) \|\|
4144	(bfloat16_is_normal(f) && sign) \|\|
4145	(bfloat16_is_zero_or_denormal(f) && !bfloat16_is_zero(f) && sign)) {
4146	float_raise(float_flag_invalid, s);
4147	return bfloat16_default_nan(s);
4148	}
4149
4150	/* frsqrt7(qNaN) = canonical NaN */
4151	if (bfloat16_is_quiet_nan(f, s)) {
4152	return bfloat16_default_nan(s);
4153	}
4154
4155	/* frsqrt7(+-0) = +-inf */
4156	if (bfloat16_is_zero(f)) {
4157	float_raise(float_flag_divbyzero, s);
4158	return bfloat16_set_sign(bfloat16_infinity, sign);
4159	}
4160
4161	/* frsqrt7(+inf) = +0 */
4162	if (bfloat16_is_infinity(f) && !sign) {
4163	return bfloat16_set_sign(bfloat16_zero, sign);
4164	}
4165
4166	/* +normal, +subnormal */
4167	uint64_t val = frsqrt7(f, exp_size, frac_size);
4168	return make_float16(val);
4169	}
4170
4171	static float16 frsqrt7_h(float16 f, float_status *s)
4172	{
4173	int exp_size = 5, frac_size = 10;
4174	bool sign = float16_is_neg(f);
4175
4176	/*
4177	* frsqrt7(sNaN) = canonical NaN
4178	* frsqrt7(-inf) = canonical NaN
4179	* frsqrt7(-normal) = canonical NaN
4180	* frsqrt7(-subnormal) = canonical NaN
4181	*/
4182	if (float16_is_signaling_nan(f, s) \|\|
4183	(float16_is_infinity(f) && sign) \|\|
4184	(float16_is_normal(f) && sign) \|\|
4185	(float16_is_zero_or_denormal(f) && !float16_is_zero(f) && sign)) {
4186	float_raise(float_flag_invalid, s);
4187	return float16_default_nan(s);
4188	}
4189
4190	/* frsqrt7(qNaN) = canonical NaN */
4191	if (float16_is_quiet_nan(f, s)) {
4192	return float16_default_nan(s);
4193	}
4194
4195	/* frsqrt7(+-0) = +-inf */
4196	if (float16_is_zero(f)) {
4197	float_raise(float_flag_divbyzero, s);
4198	return float16_set_sign(float16_infinity, sign);
4199	}
4200
4201	/* frsqrt7(+inf) = +0 */
4202	if (float16_is_infinity(f) && !sign) {
4203	return float16_set_sign(float16_zero, sign);
4204	}
4205
4206	/* +normal, +subnormal */
4207	uint64_t val = frsqrt7(f, exp_size, frac_size);
4208	return make_float16(val);
4209	}
4210
4211	static float32 frsqrt7_s(float32 f, float_status *s)
4212	{
4213	int exp_size = 8, frac_size = 23;
4214	bool sign = float32_is_neg(f);
4215
4216	/*
4217	* frsqrt7(sNaN) = canonical NaN
4218	* frsqrt7(-inf) = canonical NaN
4219	* frsqrt7(-normal) = canonical NaN
4220	* frsqrt7(-subnormal) = canonical NaN
4221	*/
4222	if (float32_is_signaling_nan(f, s) \|\|
4223	(float32_is_infinity(f) && sign) \|\|
4224	(float32_is_normal(f) && sign) \|\|
4225	(float32_is_zero_or_denormal(f) && !float32_is_zero(f) && sign)) {
4226	float_raise(float_flag_invalid, s);
4227	return float32_default_nan(s);
4228	}
4229
4230	/* frsqrt7(qNaN) = canonical NaN */
4231	if (float32_is_quiet_nan(f, s)) {
4232	return float32_default_nan(s);
4233	}
4234
4235	/* frsqrt7(+-0) = +-inf */
4236	if (float32_is_zero(f)) {
4237	float_raise(float_flag_divbyzero, s);
4238	return float32_set_sign(float32_infinity, sign);
4239	}
4240
4241	/* frsqrt7(+inf) = +0 */
4242	if (float32_is_infinity(f) && !sign) {
4243	return float32_set_sign(float32_zero, sign);
4244	}
4245
4246	/* +normal, +subnormal */
4247	uint64_t val = frsqrt7(f, exp_size, frac_size);
4248	return make_float32(val);
4249	}
4250
4251	static float64 frsqrt7_d(float64 f, float_status *s)
4252	{
4253	int exp_size = 11, frac_size = 52;
4254	bool sign = float64_is_neg(f);
4255
4256	/*
4257	* frsqrt7(sNaN) = canonical NaN
4258	* frsqrt7(-inf) = canonical NaN
4259	* frsqrt7(-normal) = canonical NaN
4260	* frsqrt7(-subnormal) = canonical NaN
4261	*/
4262	if (float64_is_signaling_nan(f, s) \|\|
4263	(float64_is_infinity(f) && sign) \|\|
4264	(float64_is_normal(f) && sign) \|\|
4265	(float64_is_zero_or_denormal(f) && !float64_is_zero(f) && sign)) {
4266	float_raise(float_flag_invalid, s);
4267	return float64_default_nan(s);
4268	}
4269
4270	/* frsqrt7(qNaN) = canonical NaN */
4271	if (float64_is_quiet_nan(f, s)) {
4272	return float64_default_nan(s);
4273	}
4274
4275	/* frsqrt7(+-0) = +-inf */
4276	if (float64_is_zero(f)) {
4277	float_raise(float_flag_divbyzero, s);
4278	return float64_set_sign(float64_infinity, sign);
4279	}
4280
4281	/* frsqrt7(+inf) = +0 */
4282	if (float64_is_infinity(f) && !sign) {
4283	return float64_set_sign(float64_zero, sign);
4284	}
4285
4286	/* +normal, +subnormal */
4287	uint64_t val = frsqrt7(f, exp_size, frac_size);
4288	return make_float64(val);
4289	}
4290
4291	RVVCALL(OPFVV1, vfrsqrt7_v_h_bf16, OP_UU_H, H2, H2, frsqrt7_h_bf16)
4292	RVVCALL(OPFVV1, vfrsqrt7_v_h, OP_UU_H, H2, H2, frsqrt7_h)
4293	RVVCALL(OPFVV1, vfrsqrt7_v_w, OP_UU_W, H4, H4, frsqrt7_s)
4294	RVVCALL(OPFVV1, vfrsqrt7_v_d, OP_UU_D, H8, H8, frsqrt7_d)
4295	GEN_VEXT_V_ENV(vfrsqrt7_v_h_bf16, 2)
4296	GEN_VEXT_V_ENV(vfrsqrt7_v_h, 2)
4297	GEN_VEXT_V_ENV(vfrsqrt7_v_w, 4)
4298	GEN_VEXT_V_ENV(vfrsqrt7_v_d, 8)
4299
4300	/*
4301	* Vector Floating-Point Reciprocal Estimate Instruction
4302	*
4303	* Adapted from riscv-v-spec recip.c:
4304	* https://github.com/riscv/riscv-v-spec/blob/master/recip.c
4305	*/
4306	static uint64_t frec7(uint64_t f, int exp_size, int frac_size,
4307	float_status *s)
4308	{
4309	uint64_t sign = extract64(f, frac_size + exp_size, 1);
4310	uint64_t exp = extract64(f, frac_size, exp_size);
4311	uint64_t frac = extract64(f, 0, frac_size);
4312
4313	const uint8_t lookup_table[] = {
4314	127, 125, 123, 121, 119, 117, 116, 114,
4315	112, 110, 109, 107, 105, 104, 102, 100,
4316	99, 97, 96, 94, 93, 91, 90, 88,
4317	87, 85, 84, 83, 81, 80, 79, 77,
4318	76, 75, 74, 72, 71, 70, 69, 68,
4319	66, 65, 64, 63, 62, 61, 60, 59,
4320	58, 57, 56, 55, 54, 53, 52, 51,
4321	50, 49, 48, 47, 46, 45, 44, 43,
4322	42, 41, 40, 40, 39, 38, 37, 36,
4323	35, 35, 34, 33, 32, 31, 31, 30,
4324	29, 28, 28, 27, 26, 25, 25, 24,
4325	23, 23, 22, 21, 21, 20, 19, 19,
4326	18, 17, 17, 16, 15, 15, 14, 14,
4327	13, 12, 12, 11, 11, 10, 9, 9,
4328	8, 8, 7, 7, 6, 5, 5, 4,
4329	4, 3, 3, 2, 2, 1, 1, 0
4330	};
4331	const int precision = 7;
4332
4333	if (exp == 0 && frac != 0) { /* subnormal */
4334	/* Normalize the subnormal. */
4335	while (extract64(frac, frac_size - 1, 1) == 0) {
4336	exp--;
4337	frac <<= 1;
4338	}
4339
4340	frac = (frac << 1) & MAKE_64BIT_MASK(0, frac_size);
4341
4342	if (exp != 0 && exp != UINT64_MAX) {
4343	/*
4344	* Overflow to inf or max value of same sign,
4345	* depending on sign and rounding mode.
4346	*/
4347	float_raise(float_flag_inexact \| float_flag_overflow, s);
4348
4349	if ((get_float_rounding_mode(s) == float_round_to_zero) \|\|
4350	((get_float_rounding_mode(s) == float_round_down) && !sign) \|\|
4351	((get_float_rounding_mode(s) == float_round_up) && sign)) {
4352	/* Return greatest/negative finite value. */
4353	return (sign << (exp_size + frac_size)) \|
4354	(MAKE_64BIT_MASK(frac_size, exp_size) - 1);
4355	} else {
4356	/* Return +-inf. */
4357	return (sign << (exp_size + frac_size)) \|
4358	MAKE_64BIT_MASK(frac_size, exp_size);
4359	}
4360	}
4361	}
4362
4363	int idx = frac >> (frac_size - precision);
4364	uint64_t out_frac = (uint64_t)(lookup_table[idx]) <<
4365	(frac_size - precision);
4366	uint64_t out_exp = 2 * MAKE_64BIT_MASK(0, exp_size - 1) + ~exp;
4367
4368	if (out_exp == 0 \|\| out_exp == UINT64_MAX) {
4369	/*
4370	* The result is subnormal, but don't raise the underflow exception,
4371	* because there's no additional loss of precision.
4372	*/
4373	out_frac = (out_frac >> 1) \| MAKE_64BIT_MASK(frac_size - 1, 1);
4374	if (out_exp == UINT64_MAX) {
4375	out_frac >>= 1;
4376	out_exp = 0;
4377	}
4378	}
4379
4380	uint64_t val = 0;
4381	val = deposit64(val, 0, frac_size, out_frac);
4382	val = deposit64(val, frac_size, exp_size, out_exp);
4383	val = deposit64(val, frac_size + exp_size, 1, sign);
4384	return val;
4385	}
4386
4387	static bfloat16 frec7_h_bf16(bfloat16 f, float_status *s)
4388	{
4389	int exp_size = 8, frac_size = 7;
4390	bool sign = bfloat16_is_neg(f);
4391
4392	/* frec7(+-inf) = +-0 */
4393	if (bfloat16_is_infinity(f)) {
4394	return bfloat16_set_sign(bfloat16_zero, sign);
4395	}
4396
4397	/* frec7(+-0) = +-inf */
4398	if (bfloat16_is_zero(f)) {
4399	float_raise(float_flag_divbyzero, s);
4400	return bfloat16_set_sign(bfloat16_infinity, sign);
4401	}
4402
4403	/* frec7(sNaN) = canonical NaN */
4404	if (bfloat16_is_signaling_nan(f, s)) {
4405	float_raise(float_flag_invalid, s);
4406	return bfloat16_default_nan(s);
4407	}
4408
4409	/* frec7(qNaN) = canonical NaN */
4410	if (bfloat16_is_quiet_nan(f, s)) {
4411	return bfloat16_default_nan(s);
4412	}
4413
4414	/* +-normal, +-subnormal */
4415	uint64_t val = frec7(f, exp_size, frac_size, s);
4416	return make_float16(val);
4417	}
4418
4419	static float16 frec7_h(float16 f, float_status *s)
4420	{
4421	int exp_size = 5, frac_size = 10;
4422	bool sign = float16_is_neg(f);
4423
4424	/* frec7(+-inf) = +-0 */
4425	if (float16_is_infinity(f)) {
4426	return float16_set_sign(float16_zero, sign);
4427	}
4428
4429	/* frec7(+-0) = +-inf */
4430	if (float16_is_zero(f)) {
4431	float_raise(float_flag_divbyzero, s);
4432	return float16_set_sign(float16_infinity, sign);
4433	}
4434
4435	/* frec7(sNaN) = canonical NaN */
4436	if (float16_is_signaling_nan(f, s)) {
4437	float_raise(float_flag_invalid, s);
4438	return float16_default_nan(s);
4439	}
4440
4441	/* frec7(qNaN) = canonical NaN */
4442	if (float16_is_quiet_nan(f, s)) {
4443	return float16_default_nan(s);
4444	}
4445
4446	/* +-normal, +-subnormal */
4447	uint64_t val = frec7(f, exp_size, frac_size, s);
4448	return make_float16(val);
4449	}
4450
4451	static float32 frec7_s(float32 f, float_status *s)
4452	{
4453	int exp_size = 8, frac_size = 23;
4454	bool sign = float32_is_neg(f);
4455
4456	/* frec7(+-inf) = +-0 */
4457	if (float32_is_infinity(f)) {
4458	return float32_set_sign(float32_zero, sign);
4459	}
4460
4461	/* frec7(+-0) = +-inf */
4462	if (float32_is_zero(f)) {
4463	float_raise(float_flag_divbyzero, s);
4464	return float32_set_sign(float32_infinity, sign);
4465	}
4466
4467	/* frec7(sNaN) = canonical NaN */
4468	if (float32_is_signaling_nan(f, s)) {
4469	float_raise(float_flag_invalid, s);
4470	return float32_default_nan(s);
4471	}
4472
4473	/* frec7(qNaN) = canonical NaN */
4474	if (float32_is_quiet_nan(f, s)) {
4475	return float32_default_nan(s);
4476	}
4477
4478	/* +-normal, +-subnormal */
4479	uint64_t val = frec7(f, exp_size, frac_size, s);
4480	return make_float32(val);
4481	}
4482
4483	static float64 frec7_d(float64 f, float_status *s)
4484	{
4485	int exp_size = 11, frac_size = 52;
4486	bool sign = float64_is_neg(f);
4487
4488	/* frec7(+-inf) = +-0 */
4489	if (float64_is_infinity(f)) {
4490	return float64_set_sign(float64_zero, sign);
4491	}
4492
4493	/* frec7(+-0) = +-inf */
4494	if (float64_is_zero(f)) {
4495	float_raise(float_flag_divbyzero, s);
4496	return float64_set_sign(float64_infinity, sign);
4497	}
4498
4499	/* frec7(sNaN) = canonical NaN */
4500	if (float64_is_signaling_nan(f, s)) {
4501	float_raise(float_flag_invalid, s);
4502	return float64_default_nan(s);
4503	}
4504
4505	/* frec7(qNaN) = canonical NaN */
4506	if (float64_is_quiet_nan(f, s)) {
4507	return float64_default_nan(s);
4508	}
4509
4510	/* +-normal, +-subnormal */
4511	uint64_t val = frec7(f, exp_size, frac_size, s);
4512	return make_float64(val);
4513	}
4514
4515	RVVCALL(OPFVV1, vfrec7_v_h_bf16, OP_UU_H, H2, H2, frec7_h_bf16)
4516	RVVCALL(OPFVV1, vfrec7_v_h, OP_UU_H, H2, H2, frec7_h)
4517	RVVCALL(OPFVV1, vfrec7_v_w, OP_UU_W, H4, H4, frec7_s)
4518	RVVCALL(OPFVV1, vfrec7_v_d, OP_UU_D, H8, H8, frec7_d)
4519	GEN_VEXT_V_ENV(vfrec7_v_h_bf16, 2)
4520	GEN_VEXT_V_ENV(vfrec7_v_h, 2)
4521	GEN_VEXT_V_ENV(vfrec7_v_w, 4)
4522	GEN_VEXT_V_ENV(vfrec7_v_d, 8)
4523
4524	/* Vector Floating-Point MIN/MAX Instructions */
4525	RVVCALL(OPFVV2, vfmin_vv_h_bf16, OP_UUU_H, H2, H2, H2, bfloat16_minimum_number)
4526	RVVCALL(OPFVV2, vfmin_vv_h, OP_UUU_H, H2, H2, H2, float16_minimum_number)
4527	RVVCALL(OPFVV2, vfmin_vv_w, OP_UUU_W, H4, H4, H4, float32_minimum_number)
4528	RVVCALL(OPFVV2, vfmin_vv_d, OP_UUU_D, H8, H8, H8, float64_minimum_number)
4529	GEN_VEXT_VV_ENV(vfmin_vv_h_bf16, 2)
4530	GEN_VEXT_VV_ENV(vfmin_vv_h, 2)
4531	GEN_VEXT_VV_ENV(vfmin_vv_w, 4)
4532	GEN_VEXT_VV_ENV(vfmin_vv_d, 8)
4533	RVVCALL(OPFVF2, vfmin_vf_h_bf16, OP_UUU_H, H2, H2, bfloat16_minimum_number)
4534	RVVCALL(OPFVF2, vfmin_vf_h, OP_UUU_H, H2, H2, float16_minimum_number)
4535	RVVCALL(OPFVF2, vfmin_vf_w, OP_UUU_W, H4, H4, float32_minimum_number)
4536	RVVCALL(OPFVF2, vfmin_vf_d, OP_UUU_D, H8, H8, float64_minimum_number)
4537	GEN_VEXT_VF(vfmin_vf_h_bf16, 2)
4538	GEN_VEXT_VF(vfmin_vf_h, 2)
4539	GEN_VEXT_VF(vfmin_vf_w, 4)
4540	GEN_VEXT_VF(vfmin_vf_d, 8)
4541
4542	RVVCALL(OPFVV2, vfmax_vv_h_bf16, OP_UUU_H, H2, H2, H2, bfloat16_maximum_number)
4543	RVVCALL(OPFVV2, vfmax_vv_h, OP_UUU_H, H2, H2, H2, float16_maximum_number)
4544	RVVCALL(OPFVV2, vfmax_vv_w, OP_UUU_W, H4, H4, H4, float32_maximum_number)
4545	RVVCALL(OPFVV2, vfmax_vv_d, OP_UUU_D, H8, H8, H8, float64_maximum_number)
4546	GEN_VEXT_VV_ENV(vfmax_vv_h_bf16, 2)
4547	GEN_VEXT_VV_ENV(vfmax_vv_h, 2)
4548	GEN_VEXT_VV_ENV(vfmax_vv_w, 4)
4549	GEN_VEXT_VV_ENV(vfmax_vv_d, 8)
4550	RVVCALL(OPFVF2, vfmax_vf_h_bf16, OP_UUU_H, H2, H2, bfloat16_maximum_number)
4551	RVVCALL(OPFVF2, vfmax_vf_h, OP_UUU_H, H2, H2, float16_maximum_number)
4552	RVVCALL(OPFVF2, vfmax_vf_w, OP_UUU_W, H4, H4, float32_maximum_number)
4553	RVVCALL(OPFVF2, vfmax_vf_d, OP_UUU_D, H8, H8, float64_maximum_number)
4554	GEN_VEXT_VF(vfmax_vf_h_bf16, 2)
4555	GEN_VEXT_VF(vfmax_vf_h, 2)
4556	GEN_VEXT_VF(vfmax_vf_w, 4)
4557	GEN_VEXT_VF(vfmax_vf_d, 8)
4558
4559	/* Vector Floating-Point Sign-Injection Instructions */
4560	static uint16_t fsgnj16(uint16_t a, uint16_t b, float_status *s)
4561	{
4562	return deposit64(b, 0, 15, a);
4563	}
4564
4565	static uint32_t fsgnj32(uint32_t a, uint32_t b, float_status *s)
4566	{
4567	return deposit64(b, 0, 31, a);
4568	}
4569
4570	static uint64_t fsgnj64(uint64_t a, uint64_t b, float_status *s)
4571	{
4572	return deposit64(b, 0, 63, a);
4573	}
4574
4575	RVVCALL(OPFVV2, vfsgnj_vv_h, OP_UUU_H, H2, H2, H2, fsgnj16)
4576	RVVCALL(OPFVV2, vfsgnj_vv_w, OP_UUU_W, H4, H4, H4, fsgnj32)
4577	RVVCALL(OPFVV2, vfsgnj_vv_d, OP_UUU_D, H8, H8, H8, fsgnj64)
4578	GEN_VEXT_VV_ENV(vfsgnj_vv_h, 2)
4579	GEN_VEXT_VV_ENV(vfsgnj_vv_w, 4)
4580	GEN_VEXT_VV_ENV(vfsgnj_vv_d, 8)
4581	RVVCALL(OPFVF2, vfsgnj_vf_h, OP_UUU_H, H2, H2, fsgnj16)
4582	RVVCALL(OPFVF2, vfsgnj_vf_w, OP_UUU_W, H4, H4, fsgnj32)
4583	RVVCALL(OPFVF2, vfsgnj_vf_d, OP_UUU_D, H8, H8, fsgnj64)
4584	GEN_VEXT_VF(vfsgnj_vf_h, 2)
4585	GEN_VEXT_VF(vfsgnj_vf_w, 4)
4586	GEN_VEXT_VF(vfsgnj_vf_d, 8)
4587
4588	static uint16_t fsgnjn16(uint16_t a, uint16_t b, float_status *s)
4589	{
4590	return deposit64(~b, 0, 15, a);
4591	}
4592
4593	static uint32_t fsgnjn32(uint32_t a, uint32_t b, float_status *s)
4594	{
4595	return deposit64(~b, 0, 31, a);
4596	}
4597
4598	static uint64_t fsgnjn64(uint64_t a, uint64_t b, float_status *s)
4599	{
4600	return deposit64(~b, 0, 63, a);
4601	}
4602
4603	RVVCALL(OPFVV2, vfsgnjn_vv_h, OP_UUU_H, H2, H2, H2, fsgnjn16)
4604	RVVCALL(OPFVV2, vfsgnjn_vv_w, OP_UUU_W, H4, H4, H4, fsgnjn32)
4605	RVVCALL(OPFVV2, vfsgnjn_vv_d, OP_UUU_D, H8, H8, H8, fsgnjn64)
4606	GEN_VEXT_VV_ENV(vfsgnjn_vv_h, 2)
4607	GEN_VEXT_VV_ENV(vfsgnjn_vv_w, 4)
4608	GEN_VEXT_VV_ENV(vfsgnjn_vv_d, 8)
4609	RVVCALL(OPFVF2, vfsgnjn_vf_h, OP_UUU_H, H2, H2, fsgnjn16)
4610	RVVCALL(OPFVF2, vfsgnjn_vf_w, OP_UUU_W, H4, H4, fsgnjn32)
4611	RVVCALL(OPFVF2, vfsgnjn_vf_d, OP_UUU_D, H8, H8, fsgnjn64)
4612	GEN_VEXT_VF(vfsgnjn_vf_h, 2)
4613	GEN_VEXT_VF(vfsgnjn_vf_w, 4)
4614	GEN_VEXT_VF(vfsgnjn_vf_d, 8)
4615
4616	static uint16_t fsgnjx16(uint16_t a, uint16_t b, float_status *s)
4617	{
4618	return deposit64(b ^ a, 0, 15, a);
4619	}
4620
4621	static uint32_t fsgnjx32(uint32_t a, uint32_t b, float_status *s)
4622	{
4623	return deposit64(b ^ a, 0, 31, a);
4624	}
4625
4626	static uint64_t fsgnjx64(uint64_t a, uint64_t b, float_status *s)
4627	{
4628	return deposit64(b ^ a, 0, 63, a);
4629	}
4630
4631	RVVCALL(OPFVV2, vfsgnjx_vv_h, OP_UUU_H, H2, H2, H2, fsgnjx16)
4632	RVVCALL(OPFVV2, vfsgnjx_vv_w, OP_UUU_W, H4, H4, H4, fsgnjx32)
4633	RVVCALL(OPFVV2, vfsgnjx_vv_d, OP_UUU_D, H8, H8, H8, fsgnjx64)
4634	GEN_VEXT_VV_ENV(vfsgnjx_vv_h, 2)
4635	GEN_VEXT_VV_ENV(vfsgnjx_vv_w, 4)
4636	GEN_VEXT_VV_ENV(vfsgnjx_vv_d, 8)
4637	RVVCALL(OPFVF2, vfsgnjx_vf_h, OP_UUU_H, H2, H2, fsgnjx16)
4638	RVVCALL(OPFVF2, vfsgnjx_vf_w, OP_UUU_W, H4, H4, fsgnjx32)
4639	RVVCALL(OPFVF2, vfsgnjx_vf_d, OP_UUU_D, H8, H8, fsgnjx64)
4640	GEN_VEXT_VF(vfsgnjx_vf_h, 2)
4641	GEN_VEXT_VF(vfsgnjx_vf_w, 4)
4642	GEN_VEXT_VF(vfsgnjx_vf_d, 8)
4643
4644	/* Vector Floating-Point Compare Instructions */
4645	#define GEN_VEXT_CMP_VV_ENV(NAME, ETYPE, H, DO_OP) \
4646	void HELPER(NAME)(void vd, void v0, void vs1, void vs2, \
4647	CPURISCVState *env, uint32_t desc) \
4648	{ \
4649	uint32_t vm = vext_vm(desc); \
4650	uint32_t vl = env->vl; \
4651	uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \
4652	uint32_t vta_all_1s = vext_vta_all_1s(desc); \
4653	uint32_t vma = vext_vma(desc); \
4654	uint32_t i; \
4655	FloatExceptionFlags pre_fflag = \
4656	get_float_exception_flags(&env->fp_status); \
4657	\
4658	VSTART_CHECK_EARLY_EXIT(env, vl); \
4659	\
4660	for (i = env->vstart; i < vl; i++) { \
4661	ETYPE s1 = ((ETYPE )vs1 + H(i)); \
4662	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
4663	if (!vm && !vext_elem_mask(v0, i)) { \
4664	/* set masked-off elements to 1s */ \
4665	if (vma) { \
4666	vext_set_elem_mask(vd, i, 1); \
4667	} \
4668	continue; \
4669	} \
4670	vext_set_elem_mask(vd, i, \
4671	DO_OP(s2, s1, &env->fp_status)); \
4672	} \
4673	env->vstart = 0; \
4674	/*
4675	* mask destination register are always tail-agnostic
4676	* set tail elements to 1s
4677	*/ \
4678	if (vta_all_1s) { \
4679	for (; i < total_elems; i++) { \
4680	vext_set_elem_mask(vd, i, 1); \
4681	} \
4682	} \
4683	riscv_cpu_check_fflags(env, pre_fflag); \
4684	}
4685
4686	GEN_VEXT_CMP_VV_ENV(vmfeq_vv_h_bf16, uint16_t, H2, bfloat16_eq_quiet)
4687	GEN_VEXT_CMP_VV_ENV(vmfeq_vv_h, uint16_t, H2, float16_eq_quiet)
4688	GEN_VEXT_CMP_VV_ENV(vmfeq_vv_w, uint32_t, H4, float32_eq_quiet)
4689	GEN_VEXT_CMP_VV_ENV(vmfeq_vv_d, uint64_t, H8, float64_eq_quiet)
4690
4691	#define GEN_VEXT_CMP_VF(NAME, ETYPE, H, DO_OP) \
4692	void HELPER(NAME)(void vd, void v0, uint64_t s1, void *vs2, \
4693	CPURISCVState *env, uint32_t desc) \
4694	{ \
4695	uint32_t vm = vext_vm(desc); \
4696	uint32_t vl = env->vl; \
4697	uint32_t total_elems = riscv_cpu_cfg(env)->vlenb << 3; \
4698	uint32_t vta_all_1s = vext_vta_all_1s(desc); \
4699	uint32_t vma = vext_vma(desc); \
4700	uint32_t i; \
4701	FloatExceptionFlags pre_fflag = \
4702	get_float_exception_flags(&env->fp_status); \
4703	\
4704	VSTART_CHECK_EARLY_EXIT(env, vl); \
4705	\
4706	for (i = env->vstart; i < vl; i++) { \
4707	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
4708	if (!vm && !vext_elem_mask(v0, i)) { \
4709	/* set masked-off elements to 1s */ \
4710	if (vma) { \
4711	vext_set_elem_mask(vd, i, 1); \
4712	} \
4713	continue; \
4714	} \
4715	vext_set_elem_mask(vd, i, \
4716	DO_OP(s2, (ETYPE)s1, &env->fp_status)); \
4717	} \
4718	env->vstart = 0; \
4719	/*
4720	* mask destination register are always tail-agnostic
4721	* set tail elements to 1s
4722	*/ \
4723	if (vta_all_1s) { \
4724	for (; i < total_elems; i++) { \
4725	vext_set_elem_mask(vd, i, 1); \
4726	} \
4727	} \
4728	riscv_cpu_check_fflags(env, pre_fflag); \
4729	}
4730
4731	GEN_VEXT_CMP_VF(vmfeq_vf_h_bf16, uint16_t, H2, bfloat16_eq_quiet)
4732	GEN_VEXT_CMP_VF(vmfeq_vf_h, uint16_t, H2, float16_eq_quiet)
4733	GEN_VEXT_CMP_VF(vmfeq_vf_w, uint32_t, H4, float32_eq_quiet)
4734	GEN_VEXT_CMP_VF(vmfeq_vf_d, uint64_t, H8, float64_eq_quiet)
4735
4736	static bool vmfne16_bf16(uint16_t a, uint16_t b, float_status *s)
4737	{
4738	FloatRelation compare = bfloat16_compare_quiet(a, b, s);
4739	return compare != float_relation_equal;
4740	}
4741
4742	static bool vmfne16(uint16_t a, uint16_t b, float_status *s)
4743	{
4744	FloatRelation compare = float16_compare_quiet(a, b, s);
4745	return compare != float_relation_equal;
4746	}
4747
4748	static bool vmfne32(uint32_t a, uint32_t b, float_status *s)
4749	{
4750	FloatRelation compare = float32_compare_quiet(a, b, s);
4751	return compare != float_relation_equal;
4752	}
4753
4754	static bool vmfne64(uint64_t a, uint64_t b, float_status *s)
4755	{
4756	FloatRelation compare = float64_compare_quiet(a, b, s);
4757	return compare != float_relation_equal;
4758	}
4759
4760	GEN_VEXT_CMP_VV_ENV(vmfne_vv_h_bf16, uint16_t, H2, vmfne16_bf16)
4761	GEN_VEXT_CMP_VV_ENV(vmfne_vv_h, uint16_t, H2, vmfne16)
4762	GEN_VEXT_CMP_VV_ENV(vmfne_vv_w, uint32_t, H4, vmfne32)
4763	GEN_VEXT_CMP_VV_ENV(vmfne_vv_d, uint64_t, H8, vmfne64)
4764	GEN_VEXT_CMP_VF(vmfne_vf_h_bf16, uint16_t, H2, vmfne16_bf16)
4765	GEN_VEXT_CMP_VF(vmfne_vf_h, uint16_t, H2, vmfne16)
4766	GEN_VEXT_CMP_VF(vmfne_vf_w, uint32_t, H4, vmfne32)
4767	GEN_VEXT_CMP_VF(vmfne_vf_d, uint64_t, H8, vmfne64)
4768
4769	GEN_VEXT_CMP_VV_ENV(vmflt_vv_h_bf16, uint16_t, H2, bfloat16_lt)
4770	GEN_VEXT_CMP_VV_ENV(vmflt_vv_h, uint16_t, H2, float16_lt)
4771	GEN_VEXT_CMP_VV_ENV(vmflt_vv_w, uint32_t, H4, float32_lt)
4772	GEN_VEXT_CMP_VV_ENV(vmflt_vv_d, uint64_t, H8, float64_lt)
4773	GEN_VEXT_CMP_VF(vmflt_vf_h_bf16, uint16_t, H2, bfloat16_lt)
4774	GEN_VEXT_CMP_VF(vmflt_vf_h, uint16_t, H2, float16_lt)
4775	GEN_VEXT_CMP_VF(vmflt_vf_w, uint32_t, H4, float32_lt)
4776	GEN_VEXT_CMP_VF(vmflt_vf_d, uint64_t, H8, float64_lt)
4777
4778	GEN_VEXT_CMP_VV_ENV(vmfle_vv_h_bf16, uint16_t, H2, bfloat16_le)
4779	GEN_VEXT_CMP_VV_ENV(vmfle_vv_h, uint16_t, H2, float16_le)
4780	GEN_VEXT_CMP_VV_ENV(vmfle_vv_w, uint32_t, H4, float32_le)
4781	GEN_VEXT_CMP_VV_ENV(vmfle_vv_d, uint64_t, H8, float64_le)
4782	GEN_VEXT_CMP_VF(vmfle_vf_h_bf16, uint16_t, H2, bfloat16_le)
4783	GEN_VEXT_CMP_VF(vmfle_vf_h, uint16_t, H2, float16_le)
4784	GEN_VEXT_CMP_VF(vmfle_vf_w, uint32_t, H4, float32_le)
4785	GEN_VEXT_CMP_VF(vmfle_vf_d, uint64_t, H8, float64_le)
4786
4787	static bool vmfgt16_bf16(uint16_t a, uint16_t b, float_status *s)
4788	{
4789	FloatRelation compare = bfloat16_compare(a, b, s);
4790	return compare == float_relation_greater;
4791	}
4792
4793	static bool vmfgt16(uint16_t a, uint16_t b, float_status *s)
4794	{
4795	FloatRelation compare = float16_compare(a, b, s);
4796	return compare == float_relation_greater;
4797	}
4798
4799	static bool vmfgt32(uint32_t a, uint32_t b, float_status *s)
4800	{
4801	FloatRelation compare = float32_compare(a, b, s);
4802	return compare == float_relation_greater;
4803	}
4804
4805	static bool vmfgt64(uint64_t a, uint64_t b, float_status *s)
4806	{
4807	FloatRelation compare = float64_compare(a, b, s);
4808	return compare == float_relation_greater;
4809	}
4810
4811	GEN_VEXT_CMP_VF(vmfgt_vf_h_bf16, uint16_t, H2, vmfgt16_bf16)
4812	GEN_VEXT_CMP_VF(vmfgt_vf_h, uint16_t, H2, vmfgt16)
4813	GEN_VEXT_CMP_VF(vmfgt_vf_w, uint32_t, H4, vmfgt32)
4814	GEN_VEXT_CMP_VF(vmfgt_vf_d, uint64_t, H8, vmfgt64)
4815
4816	static bool vmfge16_bf16(uint16_t a, uint16_t b, float_status *s)
4817	{
4818	FloatRelation compare = bfloat16_compare(a, b, s);
4819	return compare == float_relation_greater \|\|
4820	compare == float_relation_equal;
4821	}
4822
4823	static bool vmfge16(uint16_t a, uint16_t b, float_status *s)
4824	{
4825	FloatRelation compare = float16_compare(a, b, s);
4826	return compare == float_relation_greater \|\|
4827	compare == float_relation_equal;
4828	}
4829
4830	static bool vmfge32(uint32_t a, uint32_t b, float_status *s)
4831	{
4832	FloatRelation compare = float32_compare(a, b, s);
4833	return compare == float_relation_greater \|\|
4834	compare == float_relation_equal;
4835	}
4836
4837	static bool vmfge64(uint64_t a, uint64_t b, float_status *s)
4838	{
4839	FloatRelation compare = float64_compare(a, b, s);
4840	return compare == float_relation_greater \|\|
4841	compare == float_relation_equal;
4842	}
4843
4844	GEN_VEXT_CMP_VF(vmfge_vf_h_bf16, uint16_t, H2, vmfge16_bf16)
4845	GEN_VEXT_CMP_VF(vmfge_vf_h, uint16_t, H2, vmfge16)
4846	GEN_VEXT_CMP_VF(vmfge_vf_w, uint32_t, H4, vmfge32)
4847	GEN_VEXT_CMP_VF(vmfge_vf_d, uint64_t, H8, vmfge64)
4848
4849	/* Vector Floating-Point Classify Instruction */
4850	target_ulong fclass_h_bf16(uint64_t frs1)
4851	{
4852	bfloat16 f = frs1;
4853	bool sign = bfloat16_is_neg(f);
4854
4855	if (bfloat16_is_infinity(f)) {
4856	return sign ? 1 << 0 : 1 << 7;
4857	} else if (bfloat16_is_zero(f)) {
4858	return sign ? 1 << 3 : 1 << 4;
4859	} else if (bfloat16_is_zero_or_denormal(f)) {
4860	return sign ? 1 << 2 : 1 << 5;
4861	} else if (bfloat16_is_any_nan(f)) {
4862	float_status s = { }; /* for snan_bit_is_one */
4863	return bfloat16_is_quiet_nan(f, &s) ? 1 << 9 : 1 << 8;
4864	} else {
4865	return sign ? 1 << 1 : 1 << 6;
4866	}
4867	}
4868
4869	target_ulong fclass_h(uint64_t frs1)
4870	{
4871	float16 f = frs1;
4872	bool sign = float16_is_neg(f);
4873
4874	if (float16_is_infinity(f)) {
4875	return sign ? 1 << 0 : 1 << 7;
4876	} else if (float16_is_zero(f)) {
4877	return sign ? 1 << 3 : 1 << 4;
4878	} else if (float16_is_zero_or_denormal(f)) {
4879	return sign ? 1 << 2 : 1 << 5;
4880	} else if (float16_is_any_nan(f)) {
4881	float_status s = { }; /* for snan_bit_is_one */
4882	return float16_is_quiet_nan(f, &s) ? 1 << 9 : 1 << 8;
4883	} else {
4884	return sign ? 1 << 1 : 1 << 6;
4885	}
4886	}
4887
4888	target_ulong fclass_s(uint64_t frs1)
4889	{
4890	float32 f = frs1;
4891	bool sign = float32_is_neg(f);
4892
4893	if (float32_is_infinity(f)) {
4894	return sign ? 1 << 0 : 1 << 7;
4895	} else if (float32_is_zero(f)) {
4896	return sign ? 1 << 3 : 1 << 4;
4897	} else if (float32_is_zero_or_denormal(f)) {
4898	return sign ? 1 << 2 : 1 << 5;
4899	} else if (float32_is_any_nan(f)) {
4900	float_status s = { }; /* for snan_bit_is_one */
4901	return float32_is_quiet_nan(f, &s) ? 1 << 9 : 1 << 8;
4902	} else {
4903	return sign ? 1 << 1 : 1 << 6;
4904	}
4905	}
4906
4907	target_ulong fclass_d(uint64_t frs1)
4908	{
4909	float64 f = frs1;
4910	bool sign = float64_is_neg(f);
4911
4912	if (float64_is_infinity(f)) {
4913	return sign ? 1 << 0 : 1 << 7;
4914	} else if (float64_is_zero(f)) {
4915	return sign ? 1 << 3 : 1 << 4;
4916	} else if (float64_is_zero_or_denormal(f)) {
4917	return sign ? 1 << 2 : 1 << 5;
4918	} else if (float64_is_any_nan(f)) {
4919	float_status s = { }; /* for snan_bit_is_one */
4920	return float64_is_quiet_nan(f, &s) ? 1 << 9 : 1 << 8;
4921	} else {
4922	return sign ? 1 << 1 : 1 << 6;
4923	}
4924	}
4925
4926	RVVCALL(OPIVV1, vfclass_v_h_bf16, OP_UU_H, H2, H2, fclass_h_bf16)
4927	RVVCALL(OPIVV1, vfclass_v_h, OP_UU_H, H2, H2, fclass_h)
4928	RVVCALL(OPIVV1, vfclass_v_w, OP_UU_W, H4, H4, fclass_s)
4929	RVVCALL(OPIVV1, vfclass_v_d, OP_UU_D, H8, H8, fclass_d)
4930	GEN_VEXT_V(vfclass_v_h_bf16, 2)
4931	GEN_VEXT_V(vfclass_v_h, 2)
4932	GEN_VEXT_V(vfclass_v_w, 4)
4933	GEN_VEXT_V(vfclass_v_d, 8)
4934
4935	/* Vector Floating-Point Merge Instruction */
4936
4937	#define GEN_VFMERGE_VF(NAME, ETYPE, H) \
4938	void HELPER(NAME)(void vd, void v0, uint64_t s1, void *vs2, \
4939	CPURISCVState *env, uint32_t desc) \
4940	{ \
4941	uint32_t vm = vext_vm(desc); \
4942	uint32_t vl = env->vl; \
4943	uint32_t esz = sizeof(ETYPE); \
4944	uint32_t total_elems = \
4945	vext_get_total_elems(env, desc, esz); \
4946	uint32_t vta = vext_vta(desc); \
4947	uint32_t i; \
4948	\
4949	VSTART_CHECK_EARLY_EXIT(env, vl); \
4950	\
4951	for (i = env->vstart; i < vl; i++) { \
4952	ETYPE s2 = ((ETYPE )vs2 + H(i)); \
4953	((ETYPE )vd + H(i)) = \
4954	(!vm && !vext_elem_mask(v0, i) ? s2 : s1); \
4955	} \
4956	env->vstart = 0; \
4957	/* set tail elements to 1s */ \
4958	vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
4959	}
4960
4961	GEN_VFMERGE_VF(vfmerge_vfm_h, int16_t, H2)
4962	GEN_VFMERGE_VF(vfmerge_vfm_w, int32_t, H4)
4963	GEN_VFMERGE_VF(vfmerge_vfm_d, int64_t, H8)
4964
4965	/* Single-Width Floating-Point/Integer Type-Convert Instructions */
4966	/* vfcvt.xu.f.v vd, vs2, vm # Convert float to unsigned integer. */
4967	RVVCALL(OPFVV1, vfcvt_xu_f_v_h, OP_UU_H, H2, H2, float16_to_uint16)
4968	RVVCALL(OPFVV1, vfcvt_xu_f_v_w, OP_UU_W, H4, H4, float32_to_uint32)
4969	RVVCALL(OPFVV1, vfcvt_xu_f_v_d, OP_UU_D, H8, H8, float64_to_uint64)
4970	GEN_VEXT_V_ENV(vfcvt_xu_f_v_h, 2)
4971	GEN_VEXT_V_ENV(vfcvt_xu_f_v_w, 4)
4972	GEN_VEXT_V_ENV(vfcvt_xu_f_v_d, 8)
4973
4974	/* vfcvt.x.f.v vd, vs2, vm # Convert float to signed integer. */
4975	RVVCALL(OPFVV1, vfcvt_x_f_v_h, OP_UU_H, H2, H2, float16_to_int16)
4976	RVVCALL(OPFVV1, vfcvt_x_f_v_w, OP_UU_W, H4, H4, float32_to_int32)
4977	RVVCALL(OPFVV1, vfcvt_x_f_v_d, OP_UU_D, H8, H8, float64_to_int64)
4978	GEN_VEXT_V_ENV(vfcvt_x_f_v_h, 2)
4979	GEN_VEXT_V_ENV(vfcvt_x_f_v_w, 4)
4980	GEN_VEXT_V_ENV(vfcvt_x_f_v_d, 8)
4981
4982	/* vfcvt.f.xu.v vd, vs2, vm # Convert unsigned integer to float. */
4983	RVVCALL(OPFVV1, vfcvt_f_xu_v_h, OP_UU_H, H2, H2, uint16_to_float16)
4984	RVVCALL(OPFVV1, vfcvt_f_xu_v_w, OP_UU_W, H4, H4, uint32_to_float32)
4985	RVVCALL(OPFVV1, vfcvt_f_xu_v_d, OP_UU_D, H8, H8, uint64_to_float64)
4986	GEN_VEXT_V_ENV(vfcvt_f_xu_v_h, 2)
4987	GEN_VEXT_V_ENV(vfcvt_f_xu_v_w, 4)
4988	GEN_VEXT_V_ENV(vfcvt_f_xu_v_d, 8)
4989
4990	/* vfcvt.f.x.v vd, vs2, vm # Convert integer to float. */
4991	RVVCALL(OPFVV1, vfcvt_f_x_v_h, OP_UU_H, H2, H2, int16_to_float16)
4992	RVVCALL(OPFVV1, vfcvt_f_x_v_w, OP_UU_W, H4, H4, int32_to_float32)
4993	RVVCALL(OPFVV1, vfcvt_f_x_v_d, OP_UU_D, H8, H8, int64_to_float64)
4994	GEN_VEXT_V_ENV(vfcvt_f_x_v_h, 2)
4995	GEN_VEXT_V_ENV(vfcvt_f_x_v_w, 4)
4996	GEN_VEXT_V_ENV(vfcvt_f_x_v_d, 8)
4997
4998	/* Widening Floating-Point/Integer Type-Convert Instructions */
4999	/* (TD, T2, TX2) */
5000	#define WOP_UU_B uint16_t, uint8_t, uint8_t

Showing first 5,000 of 5,908 lines. View raw