tcg-target.c.inc · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

inc 3,226 lines 94.4 KB

Raw

1	/*
2	* Tiny Code Generator for QEMU
3	*
4	* Copyright (c) 2018 SiFive, Inc
5	* Copyright (c) 2008-2009 Arnaud Patard <arnaud.patard@rtp-net.org>
6	* Copyright (c) 2009 Aurelien Jarno <aurelien@aurel32.net>
7	* Copyright (c) 2008 Fabrice Bellard
8	*
9	* Based on i386/tcg-target.c and mips/tcg-target.c
10	*
11	* Permission is hereby granted, free of charge, to any person obtaining a copy
12	* of this software and associated documentation files (the "Software"), to deal
13	* in the Software without restriction, including without limitation the rights
14	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15	* copies of the Software, and to permit persons to whom the Software is
16	* furnished to do so, subject to the following conditions:
17	*
18	* The above copyright notice and this permission notice shall be included in
19	* all copies or substantial portions of the Software.
20	*
21	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
27	* THE SOFTWARE.
28	*/
29
30	/* Used for function call generation. */
31	#define TCG_REG_CALL_STACK TCG_REG_SP
32	#define TCG_TARGET_STACK_ALIGN 16
33	#define TCG_TARGET_CALL_STACK_OFFSET 0
34	#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL
35	#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL
36	#define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL
37	#define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL
38
39	#ifdef CONFIG_DEBUG_TCG
40	static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
41	"zero", "ra", "sp", "gp", "tp", "t0", "t1", "t2",
42	"s0", "s1", "a0", "a1", "a2", "a3", "a4", "a5",
43	"a6", "a7", "s2", "s3", "s4", "s5", "s6", "s7",
44	"s8", "s9", "s10", "s11", "t3", "t4", "t5", "t6",
45	"v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
46	"v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
47	"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
48	"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
49	};
50	#endif
51
52	static const int tcg_target_reg_alloc_order[] = {
53	/* Call saved registers */
54	/* TCG_REG_S0 reserved for TCG_AREG0 */
55	TCG_REG_S1,
56	TCG_REG_S2,
57	TCG_REG_S3,
58	TCG_REG_S4,
59	TCG_REG_S5,
60	TCG_REG_S6,
61	TCG_REG_S7,
62	TCG_REG_S8,
63	TCG_REG_S9,
64	TCG_REG_S10,
65	TCG_REG_S11,
66
67	/* Call clobbered registers */
68	TCG_REG_T0,
69	TCG_REG_T1,
70	TCG_REG_T2,
71	TCG_REG_T3,
72	TCG_REG_T4,
73	TCG_REG_T5,
74	TCG_REG_T6,
75
76	/* Argument registers */
77	TCG_REG_A0,
78	TCG_REG_A1,
79	TCG_REG_A2,
80	TCG_REG_A3,
81	TCG_REG_A4,
82	TCG_REG_A5,
83	TCG_REG_A6,
84	TCG_REG_A7,
85
86	/* Vector registers and TCG_REG_V0 reserved for mask. */
87	TCG_REG_V1, TCG_REG_V2, TCG_REG_V3, TCG_REG_V4,
88	TCG_REG_V5, TCG_REG_V6, TCG_REG_V7, TCG_REG_V8,
89	TCG_REG_V9, TCG_REG_V10, TCG_REG_V11, TCG_REG_V12,
90	TCG_REG_V13, TCG_REG_V14, TCG_REG_V15, TCG_REG_V16,
91	TCG_REG_V17, TCG_REG_V18, TCG_REG_V19, TCG_REG_V20,
92	TCG_REG_V21, TCG_REG_V22, TCG_REG_V23, TCG_REG_V24,
93	TCG_REG_V25, TCG_REG_V26, TCG_REG_V27, TCG_REG_V28,
94	TCG_REG_V29, TCG_REG_V30, TCG_REG_V31,
95	};
96
97	static const int tcg_target_call_iarg_regs[] = {
98	TCG_REG_A0,
99	TCG_REG_A1,
100	TCG_REG_A2,
101	TCG_REG_A3,
102	TCG_REG_A4,
103	TCG_REG_A5,
104	TCG_REG_A6,
105	TCG_REG_A7,
106	};
107
108	static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot)
109	{
110	tcg_debug_assert(kind == TCG_CALL_RET_NORMAL);
111	tcg_debug_assert(slot >= 0 && slot <= 1);
112	return TCG_REG_A0 + slot;
113	}
114
115	#define TCG_CT_CONST_S12 0x100
116	#define TCG_CT_CONST_M12 0x200
117	#define TCG_CT_CONST_S5 0x400
118	#define TCG_CT_CONST_CMP_VI 0x800
119
120	#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32)
121	#define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32)
122	#define ALL_DVECTOR_REG_GROUPS 0x5555555500000000
123	#define ALL_QVECTOR_REG_GROUPS 0x1111111100000000
124
125	#define sextreg sextract64
126
127	/*
128	* RISC-V Base ISA opcodes (IM)
129	*/
130
131	#define V_OPIVV (0x0 << 12)
132	#define V_OPFVV (0x1 << 12)
133	#define V_OPMVV (0x2 << 12)
134	#define V_OPIVI (0x3 << 12)
135	#define V_OPIVX (0x4 << 12)
136	#define V_OPFVF (0x5 << 12)
137	#define V_OPMVX (0x6 << 12)
138	#define V_OPCFG (0x7 << 12)
139
140	/* NF <= 7 && NF >= 0 */
141	#define V_NF(x) (x << 29)
142	#define V_UNIT_STRIDE (0x0 << 20)
143	#define V_UNIT_STRIDE_WHOLE_REG (0x8 << 20)
144
145	typedef enum {
146	VLMUL_M1 = 0, /* LMUL=1 */
147	VLMUL_M2, /* LMUL=2 */
148	VLMUL_M4, /* LMUL=4 */
149	VLMUL_M8, /* LMUL=8 */
150	VLMUL_RESERVED,
151	VLMUL_MF8, /* LMUL=1/8 */
152	VLMUL_MF4, /* LMUL=1/4 */
153	VLMUL_MF2, /* LMUL=1/2 */
154	} RISCVVlmul;
155
156	typedef enum {
157	OPC_ADD = 0x33,
158	OPC_ADDI = 0x13,
159	OPC_AND = 0x7033,
160	OPC_ANDI = 0x7013,
161	OPC_AUIPC = 0x17,
162	OPC_BEQ = 0x63,
163	OPC_BEXTI = 0x48005013,
164	OPC_BGE = 0x5063,
165	OPC_BGEU = 0x7063,
166	OPC_BLT = 0x4063,
167	OPC_BLTU = 0x6063,
168	OPC_BNE = 0x1063,
169	OPC_DIV = 0x2004033,
170	OPC_DIVU = 0x2005033,
171	OPC_JAL = 0x6f,
172	OPC_JALR = 0x67,
173	OPC_LB = 0x3,
174	OPC_LBU = 0x4003,
175	OPC_LD = 0x3003,
176	OPC_LH = 0x1003,
177	OPC_LHU = 0x5003,
178	OPC_LUI = 0x37,
179	OPC_LW = 0x2003,
180	OPC_LWU = 0x6003,
181	OPC_MUL = 0x2000033,
182	OPC_MULH = 0x2001033,
183	OPC_MULHSU = 0x2002033,
184	OPC_MULHU = 0x2003033,
185	OPC_OR = 0x6033,
186	OPC_ORI = 0x6013,
187	OPC_REM = 0x2006033,
188	OPC_REMU = 0x2007033,
189	OPC_SB = 0x23,
190	OPC_SD = 0x3023,
191	OPC_SH = 0x1023,
192	OPC_SLL = 0x1033,
193	OPC_SLLI = 0x1013,
194	OPC_SLT = 0x2033,
195	OPC_SLTI = 0x2013,
196	OPC_SLTIU = 0x3013,
197	OPC_SLTU = 0x3033,
198	OPC_SRA = 0x40005033,
199	OPC_SRAI = 0x40005013,
200	OPC_SRL = 0x5033,
201	OPC_SRLI = 0x5013,
202	OPC_SUB = 0x40000033,
203	OPC_SW = 0x2023,
204	OPC_XOR = 0x4033,
205	OPC_XORI = 0x4013,
206
207	OPC_ADDIW = 0x1b,
208	OPC_ADDW = 0x3b,
209	OPC_DIVUW = 0x200503b,
210	OPC_DIVW = 0x200403b,
211	OPC_MULW = 0x200003b,
212	OPC_REMUW = 0x200703b,
213	OPC_REMW = 0x200603b,
214	OPC_SLLIW = 0x101b,
215	OPC_SLLW = 0x103b,
216	OPC_SRAIW = 0x4000501b,
217	OPC_SRAW = 0x4000503b,
218	OPC_SRLIW = 0x501b,
219	OPC_SRLW = 0x503b,
220	OPC_SUBW = 0x4000003b,
221
222	OPC_FENCE = 0x0000000f,
223	OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */
224
225	/* Zba: Bit manipulation extension, address generation */
226	OPC_ADD_UW = 0x0800003b,
227
228	/* Zbb: Bit manipulation extension, basic bit manipulation */
229	OPC_ANDN = 0x40007033,
230	OPC_CLZ = 0x60001013,
231	OPC_CLZW = 0x6000101b,
232	OPC_CPOP = 0x60201013,
233	OPC_CPOPW = 0x6020101b,
234	OPC_CTZ = 0x60101013,
235	OPC_CTZW = 0x6010101b,
236	OPC_MAX = 0x0a006033,
237	OPC_MAXU = 0x0a007033,
238	OPC_MIN = 0x0a004033,
239	OPC_MINU = 0x0a005033,
240	OPC_ORN = 0x40006033,
241	OPC_REV8 = 0x6b805013,
242	OPC_ROL = 0x60001033,
243	OPC_ROLW = 0x6000103b,
244	OPC_ROR = 0x60005033,
245	OPC_RORW = 0x6000503b,
246	OPC_RORI = 0x60005013,
247	OPC_RORIW = 0x6000501b,
248	OPC_SEXT_B = 0x60401013,
249	OPC_SEXT_H = 0x60501013,
250	OPC_XNOR = 0x40004033,
251	OPC_ZEXT_H = 0x0800403b,
252
253	/* Zbkb: Bit Manipulation for Cryptography */
254	OPC_BREV8 = 0x68705013,
255
256	/* Zicond: integer conditional operations */
257	OPC_CZERO_EQZ = 0x0e005033,
258	OPC_CZERO_NEZ = 0x0e007033,
259
260	/* V: Vector extension 1.0 */
261	OPC_VSETVLI = 0x57 \| V_OPCFG,
262	OPC_VSETIVLI = 0xc0000057 \| V_OPCFG,
263	OPC_VSETVL = 0x80000057 \| V_OPCFG,
264
265	OPC_VLE8_V = 0x7 \| V_UNIT_STRIDE,
266	OPC_VLE16_V = 0x5007 \| V_UNIT_STRIDE,
267	OPC_VLE32_V = 0x6007 \| V_UNIT_STRIDE,
268	OPC_VLE64_V = 0x7007 \| V_UNIT_STRIDE,
269	OPC_VSE8_V = 0x27 \| V_UNIT_STRIDE,
270	OPC_VSE16_V = 0x5027 \| V_UNIT_STRIDE,
271	OPC_VSE32_V = 0x6027 \| V_UNIT_STRIDE,
272	OPC_VSE64_V = 0x7027 \| V_UNIT_STRIDE,
273
274	OPC_VL1RE64_V = 0x2007007 \| V_UNIT_STRIDE_WHOLE_REG \| V_NF(0),
275	OPC_VL2RE64_V = 0x2007007 \| V_UNIT_STRIDE_WHOLE_REG \| V_NF(1),
276	OPC_VL4RE64_V = 0x2007007 \| V_UNIT_STRIDE_WHOLE_REG \| V_NF(3),
277	OPC_VL8RE64_V = 0x2007007 \| V_UNIT_STRIDE_WHOLE_REG \| V_NF(7),
278
279	OPC_VS1R_V = 0x2000027 \| V_UNIT_STRIDE_WHOLE_REG \| V_NF(0),
280	OPC_VS2R_V = 0x2000027 \| V_UNIT_STRIDE_WHOLE_REG \| V_NF(1),
281	OPC_VS4R_V = 0x2000027 \| V_UNIT_STRIDE_WHOLE_REG \| V_NF(3),
282	OPC_VS8R_V = 0x2000027 \| V_UNIT_STRIDE_WHOLE_REG \| V_NF(7),
283
284	OPC_VMERGE_VIM = 0x5c000057 \| V_OPIVI,
285	OPC_VMERGE_VVM = 0x5c000057 \| V_OPIVV,
286
287	OPC_VADD_VV = 0x57 \| V_OPIVV,
288	OPC_VADD_VI = 0x57 \| V_OPIVI,
289	OPC_VSUB_VV = 0x8000057 \| V_OPIVV,
290	OPC_VRSUB_VI = 0xc000057 \| V_OPIVI,
291	OPC_VAND_VV = 0x24000057 \| V_OPIVV,
292	OPC_VAND_VI = 0x24000057 \| V_OPIVI,
293	OPC_VOR_VV = 0x28000057 \| V_OPIVV,
294	OPC_VOR_VI = 0x28000057 \| V_OPIVI,
295	OPC_VXOR_VV = 0x2c000057 \| V_OPIVV,
296	OPC_VXOR_VI = 0x2c000057 \| V_OPIVI,
297
298	OPC_VMUL_VV = 0x94000057 \| V_OPMVV,
299	OPC_VSADD_VV = 0x84000057 \| V_OPIVV,
300	OPC_VSADD_VI = 0x84000057 \| V_OPIVI,
301	OPC_VSSUB_VV = 0x8c000057 \| V_OPIVV,
302	OPC_VSSUB_VI = 0x8c000057 \| V_OPIVI,
303	OPC_VSADDU_VV = 0x80000057 \| V_OPIVV,
304	OPC_VSADDU_VI = 0x80000057 \| V_OPIVI,
305	OPC_VSSUBU_VV = 0x88000057 \| V_OPIVV,
306	OPC_VSSUBU_VI = 0x88000057 \| V_OPIVI,
307
308	OPC_VMAX_VV = 0x1c000057 \| V_OPIVV,
309	OPC_VMAX_VI = 0x1c000057 \| V_OPIVI,
310	OPC_VMAXU_VV = 0x18000057 \| V_OPIVV,
311	OPC_VMAXU_VI = 0x18000057 \| V_OPIVI,
312	OPC_VMIN_VV = 0x14000057 \| V_OPIVV,
313	OPC_VMIN_VI = 0x14000057 \| V_OPIVI,
314	OPC_VMINU_VV = 0x10000057 \| V_OPIVV,
315	OPC_VMINU_VI = 0x10000057 \| V_OPIVI,
316
317	OPC_VMSEQ_VV = 0x60000057 \| V_OPIVV,
318	OPC_VMSEQ_VI = 0x60000057 \| V_OPIVI,
319	OPC_VMSEQ_VX = 0x60000057 \| V_OPIVX,
320	OPC_VMSNE_VV = 0x64000057 \| V_OPIVV,
321	OPC_VMSNE_VI = 0x64000057 \| V_OPIVI,
322	OPC_VMSNE_VX = 0x64000057 \| V_OPIVX,
323
324	OPC_VMSLTU_VV = 0x68000057 \| V_OPIVV,
325	OPC_VMSLTU_VX = 0x68000057 \| V_OPIVX,
326	OPC_VMSLT_VV = 0x6c000057 \| V_OPIVV,
327	OPC_VMSLT_VX = 0x6c000057 \| V_OPIVX,
328	OPC_VMSLEU_VV = 0x70000057 \| V_OPIVV,
329	OPC_VMSLEU_VX = 0x70000057 \| V_OPIVX,
330	OPC_VMSLE_VV = 0x74000057 \| V_OPIVV,
331	OPC_VMSLE_VX = 0x74000057 \| V_OPIVX,
332
333	OPC_VMSLEU_VI = 0x70000057 \| V_OPIVI,
334	OPC_VMSLE_VI = 0x74000057 \| V_OPIVI,
335	OPC_VMSGTU_VI = 0x78000057 \| V_OPIVI,
336	OPC_VMSGTU_VX = 0x78000057 \| V_OPIVX,
337	OPC_VMSGT_VI = 0x7c000057 \| V_OPIVI,
338	OPC_VMSGT_VX = 0x7c000057 \| V_OPIVX,
339
340	OPC_VSLL_VV = 0x94000057 \| V_OPIVV,
341	OPC_VSLL_VI = 0x94000057 \| V_OPIVI,
342	OPC_VSLL_VX = 0x94000057 \| V_OPIVX,
343	OPC_VSRL_VV = 0xa0000057 \| V_OPIVV,
344	OPC_VSRL_VI = 0xa0000057 \| V_OPIVI,
345	OPC_VSRL_VX = 0xa0000057 \| V_OPIVX,
346	OPC_VSRA_VV = 0xa4000057 \| V_OPIVV,
347	OPC_VSRA_VI = 0xa4000057 \| V_OPIVI,
348	OPC_VSRA_VX = 0xa4000057 \| V_OPIVX,
349
350	OPC_VMV_V_V = 0x5e000057 \| V_OPIVV,
351	OPC_VMV_V_I = 0x5e000057 \| V_OPIVI,
352	OPC_VMV_V_X = 0x5e000057 \| V_OPIVX,
353
354	OPC_VMVNR_V = 0x9e000057 \| V_OPIVI,
355	} RISCVInsn;
356
357	static const struct {
358	RISCVInsn op;
359	bool swap;
360	} tcg_cmpcond_to_rvv_vv[] = {
361	[TCG_COND_EQ] = { OPC_VMSEQ_VV, false },
362	[TCG_COND_NE] = { OPC_VMSNE_VV, false },
363	[TCG_COND_LT] = { OPC_VMSLT_VV, false },
364	[TCG_COND_GE] = { OPC_VMSLE_VV, true },
365	[TCG_COND_GT] = { OPC_VMSLT_VV, true },
366	[TCG_COND_LE] = { OPC_VMSLE_VV, false },
367	[TCG_COND_LTU] = { OPC_VMSLTU_VV, false },
368	[TCG_COND_GEU] = { OPC_VMSLEU_VV, true },
369	[TCG_COND_GTU] = { OPC_VMSLTU_VV, true },
370	[TCG_COND_LEU] = { OPC_VMSLEU_VV, false }
371	};
372
373	static const struct {
374	RISCVInsn op;
375	int min;
376	int max;
377	bool adjust;
378	} tcg_cmpcond_to_rvv_vi[] = {
379	[TCG_COND_EQ] = { OPC_VMSEQ_VI, -16, 15, false },
380	[TCG_COND_NE] = { OPC_VMSNE_VI, -16, 15, false },
381	[TCG_COND_GT] = { OPC_VMSGT_VI, -16, 15, false },
382	[TCG_COND_LE] = { OPC_VMSLE_VI, -16, 15, false },
383	[TCG_COND_LT] = { OPC_VMSLE_VI, -15, 16, true },
384	[TCG_COND_GE] = { OPC_VMSGT_VI, -15, 16, true },
385	[TCG_COND_LEU] = { OPC_VMSLEU_VI, 0, 15, false },
386	[TCG_COND_GTU] = { OPC_VMSGTU_VI, 0, 15, false },
387	[TCG_COND_LTU] = { OPC_VMSLEU_VI, 1, 16, true },
388	[TCG_COND_GEU] = { OPC_VMSGTU_VI, 1, 16, true },
389	};
390
391	/* test if a constant matches the constraint */
392	static bool tcg_target_const_match(int64_t val, int ct,
393	TCGType type, TCGCond cond, int vece)
394	{
395	if (ct & TCG_CT_CONST) {
396	return 1;
397	}
398	if (type >= TCG_TYPE_V64) {
399	/* Val is replicated by VECE; extract the highest element. */
400	val >>= (-8 << vece) & 63;
401	}
402	/*
403	* Sign extended from 12 bits: [-0x800, 0x7ff].
404	* Used for most arithmetic, as this is the isa field.
405	*/
406	if ((ct & TCG_CT_CONST_S12) && val >= -0x800 && val <= 0x7ff) {
407	return 1;
408	}
409	/*
410	* Sign extended from 12 bits, +/- matching: [-0x7ff, 0x7ff].
411	* Used by movcond, which may need the negative value,
412	* and requires the modified constant to be representable.
413	*/
414	if ((ct & TCG_CT_CONST_M12) && val >= -0x7ff && val <= 0x7ff) {
415	return 1;
416	}
417	/*
418	* Sign extended from 5 bits: [-0x10, 0x0f].
419	* Used for vector-immediate.
420	*/
421	if ((ct & TCG_CT_CONST_S5) && val >= -0x10 && val <= 0x0f) {
422	return 1;
423	}
424	/*
425	* Used for vector compare OPIVI instructions.
426	*/
427	if ((ct & TCG_CT_CONST_CMP_VI) &&
428	val >= tcg_cmpcond_to_rvv_vi[cond].min &&
429	val <= tcg_cmpcond_to_rvv_vi[cond].max) {
430	return true;
431	}
432	return 0;
433	}
434
435	/*
436	* RISC-V immediate and instruction encoders (excludes 16-bit RVC)
437	*/
438
439	/* Type-R */
440
441	static int32_t encode_r(RISCVInsn opc, TCGReg rd, TCGReg rs1, TCGReg rs2)
442	{
443	return opc \| (rd & 0x1f) << 7 \| (rs1 & 0x1f) << 15 \| (rs2 & 0x1f) << 20;
444	}
445
446	/* Type-I */
447
448	static int32_t encode_imm12(uint32_t imm)
449	{
450	return (imm & 0xfff) << 20;
451	}
452
453	static int32_t encode_i(RISCVInsn opc, TCGReg rd, TCGReg rs1, uint32_t imm)
454	{
455	return opc \| (rd & 0x1f) << 7 \| (rs1 & 0x1f) << 15 \| encode_imm12(imm);
456	}
457
458	/* Type-S */
459
460	static int32_t encode_simm12(uint32_t imm)
461	{
462	int32_t ret = 0;
463
464	ret \|= (imm & 0xFE0) << 20;
465	ret \|= (imm & 0x1F) << 7;
466
467	return ret;
468	}
469
470	static int32_t encode_s(RISCVInsn opc, TCGReg rs1, TCGReg rs2, uint32_t imm)
471	{
472	return opc \| (rs1 & 0x1f) << 15 \| (rs2 & 0x1f) << 20 \| encode_simm12(imm);
473	}
474
475	/* Type-SB */
476
477	static int32_t encode_sbimm12(uint32_t imm)
478	{
479	int32_t ret = 0;
480
481	ret \|= (imm & 0x1000) << 19;
482	ret \|= (imm & 0x7e0) << 20;
483	ret \|= (imm & 0x1e) << 7;
484	ret \|= (imm & 0x800) >> 4;
485
486	return ret;
487	}
488
489	static int32_t encode_sb(RISCVInsn opc, TCGReg rs1, TCGReg rs2, uint32_t imm)
490	{
491	return opc \| (rs1 & 0x1f) << 15 \| (rs2 & 0x1f) << 20 \| encode_sbimm12(imm);
492	}
493
494	/* Type-U */
495
496	static int32_t encode_uimm20(uint32_t imm)
497	{
498	return imm & 0xfffff000;
499	}
500
501	static int32_t encode_u(RISCVInsn opc, TCGReg rd, uint32_t imm)
502	{
503	return opc \| (rd & 0x1f) << 7 \| encode_uimm20(imm);
504	}
505
506	/* Type-UJ */
507
508	static int32_t encode_ujimm20(uint32_t imm)
509	{
510	int32_t ret = 0;
511
512	ret \|= (imm & 0x0007fe) << (21 - 1);
513	ret \|= (imm & 0x000800) << (20 - 11);
514	ret \|= (imm & 0x0ff000) << (12 - 12);
515	ret \|= (imm & 0x100000) << (31 - 20);
516
517	return ret;
518	}
519
520	static int32_t encode_uj(RISCVInsn opc, TCGReg rd, uint32_t imm)
521	{
522	return opc \| (rd & 0x1f) << 7 \| encode_ujimm20(imm);
523	}
524
525
526	/* Type-OPIVI */
527
528	static int32_t encode_vi(RISCVInsn opc, TCGReg rd, int32_t imm,
529	TCGReg vs2, bool vm)
530	{
531	return opc \| (rd & 0x1f) << 7 \| (imm & 0x1f) << 15 \|
532	(vs2 & 0x1f) << 20 \| (vm << 25);
533	}
534
535	/* Type-OPIVV/OPMVV/OPIVX/OPMVX, Vector load and store */
536
537	static int32_t encode_v(RISCVInsn opc, TCGReg d, TCGReg s1,
538	TCGReg s2, bool vm)
539	{
540	return opc \| (d & 0x1f) << 7 \| (s1 & 0x1f) << 15 \|
541	(s2 & 0x1f) << 20 \| (vm << 25);
542	}
543
544	/* Vector vtype */
545
546	static uint32_t encode_vtype(bool vta, bool vma,
547	MemOp vsew, RISCVVlmul vlmul)
548	{
549	return vma << 7 \| vta << 6 \| vsew << 3 \| vlmul;
550	}
551
552	static int32_t encode_vset(RISCVInsn opc, TCGReg rd,
553	TCGArg rs1, uint32_t vtype)
554	{
555	return opc \| (rd & 0x1f) << 7 \| (rs1 & 0x1f) << 15 \| (vtype & 0x7ff) << 20;
556	}
557
558	static int32_t encode_vseti(RISCVInsn opc, TCGReg rd,
559	uint32_t uimm, uint32_t vtype)
560	{
561	return opc \| (rd & 0x1f) << 7 \| (uimm & 0x1f) << 15 \| (vtype & 0x3ff) << 20;
562	}
563
564	/*
565	* RISC-V instruction emitters
566	*/
567
568	static void tcg_out_opc_reg(TCGContext *s, RISCVInsn opc,
569	TCGReg rd, TCGReg rs1, TCGReg rs2)
570	{
571	tcg_out32(s, encode_r(opc, rd, rs1, rs2));
572	}
573
574	static void tcg_out_opc_imm(TCGContext *s, RISCVInsn opc,
575	TCGReg rd, TCGReg rs1, TCGArg imm)
576	{
577	tcg_out32(s, encode_i(opc, rd, rs1, imm));
578	}
579
580	static void tcg_out_opc_store(TCGContext *s, RISCVInsn opc,
581	TCGReg rs1, TCGReg rs2, uint32_t imm)
582	{
583	tcg_out32(s, encode_s(opc, rs1, rs2, imm));
584	}
585
586	static void tcg_out_opc_branch(TCGContext *s, RISCVInsn opc,
587	TCGReg rs1, TCGReg rs2, uint32_t imm)
588	{
589	tcg_out32(s, encode_sb(opc, rs1, rs2, imm));
590	}
591
592	static void tcg_out_opc_upper(TCGContext *s, RISCVInsn opc,
593	TCGReg rd, uint32_t imm)
594	{
595	tcg_out32(s, encode_u(opc, rd, imm));
596	}
597
598	static void tcg_out_opc_jump(TCGContext *s, RISCVInsn opc,
599	TCGReg rd, uint32_t imm)
600	{
601	tcg_out32(s, encode_uj(opc, rd, imm));
602	}
603
604	static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
605	{
606	int i;
607	for (i = 0; i < count; ++i) {
608	p[i] = OPC_NOP;
609	}
610	}
611
612	/*
613	* Relocations
614	*/
615
616	static bool reloc_sbimm12(tcg_insn_unit src_rw, const tcg_insn_unit target)
617	{
618	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
619	intptr_t offset = (intptr_t)target - (intptr_t)src_rx;
620
621	tcg_debug_assert((offset & 1) == 0);
622	if (offset == sextreg(offset, 0, 12)) {
623	*src_rw \|= encode_sbimm12(offset);
624	return true;
625	}
626
627	return false;
628	}
629
630	static bool reloc_jimm20(tcg_insn_unit src_rw, const tcg_insn_unit target)
631	{
632	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
633	intptr_t offset = (intptr_t)target - (intptr_t)src_rx;
634
635	tcg_debug_assert((offset & 1) == 0);
636	if (offset == sextreg(offset, 0, 20)) {
637	*src_rw \|= encode_ujimm20(offset);
638	return true;
639	}
640
641	return false;
642	}
643
644	static bool reloc_call(tcg_insn_unit src_rw, const tcg_insn_unit target)
645	{
646	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
647	intptr_t offset = (intptr_t)target - (intptr_t)src_rx;
648	int32_t lo = sextreg(offset, 0, 12);
649	int32_t hi = offset - lo;
650
651	if (offset == hi + lo) {
652	src_rw[0] \|= encode_uimm20(hi);
653	src_rw[1] \|= encode_imm12(lo);
654	return true;
655	}
656
657	return false;
658	}
659
660	static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
661	intptr_t value, intptr_t addend)
662	{
663	tcg_debug_assert(addend == 0);
664	switch (type) {
665	case R_RISCV_BRANCH:
666	return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value);
667	case R_RISCV_JAL:
668	return reloc_jimm20(code_ptr, (tcg_insn_unit *)value);
669	case R_RISCV_CALL:
670	return reloc_call(code_ptr, (tcg_insn_unit *)value);
671	default:
672	g_assert_not_reached();
673	}
674	}
675
676	/*
677	* RISC-V vector instruction emitters
678	*/
679
680	/*
681	* Vector registers uses the same 5 lower bits as GPR registers,
682	* and vm=0 (vm = false) means vector masking ENABLED.
683	* With RVV 1.0, vs2 is the first operand, while rs1/imm is the
684	* second operand.
685	*/
686	static void tcg_out_opc_vv(TCGContext *s, RISCVInsn opc,
687	TCGReg vd, TCGReg vs2, TCGReg vs1)
688	{
689	tcg_out32(s, encode_v(opc, vd, vs1, vs2, true));
690	}
691
692	static void tcg_out_opc_vx(TCGContext *s, RISCVInsn opc,
693	TCGReg vd, TCGReg vs2, TCGReg rs1)
694	{
695	tcg_out32(s, encode_v(opc, vd, rs1, vs2, true));
696	}
697
698	static void tcg_out_opc_vi(TCGContext *s, RISCVInsn opc,
699	TCGReg vd, TCGReg vs2, int32_t imm)
700	{
701	tcg_out32(s, encode_vi(opc, vd, imm, vs2, true));
702	}
703
704	static void tcg_out_opc_vv_vi(TCGContext *s, RISCVInsn o_vv, RISCVInsn o_vi,
705	TCGReg vd, TCGReg vs2, TCGArg vi1, int c_vi1)
706	{
707	if (c_vi1) {
708	tcg_out_opc_vi(s, o_vi, vd, vs2, vi1);
709	} else {
710	tcg_out_opc_vv(s, o_vv, vd, vs2, vi1);
711	}
712	}
713
714	static void tcg_out_opc_vim_mask(TCGContext *s, RISCVInsn opc, TCGReg vd,
715	TCGReg vs2, int32_t imm)
716	{
717	tcg_out32(s, encode_vi(opc, vd, imm, vs2, false));
718	}
719
720	static void tcg_out_opc_vvm_mask(TCGContext *s, RISCVInsn opc, TCGReg vd,
721	TCGReg vs2, TCGReg vs1)
722	{
723	tcg_out32(s, encode_v(opc, vd, vs1, vs2, false));
724	}
725
726	typedef struct VsetCache {
727	uint32_t movi_insn;
728	uint32_t vset_insn;
729	} VsetCache;
730
731	static VsetCache riscv_vset_cache[3][4];
732
733	static void set_vtype(TCGContext *s, TCGType type, MemOp vsew)
734	{
735	const VsetCache *p = &riscv_vset_cache[type - TCG_TYPE_V64][vsew];
736
737	s->riscv_cur_type = type;
738	s->riscv_cur_vsew = vsew;
739
740	if (p->movi_insn) {
741	tcg_out32(s, p->movi_insn);
742	}
743	tcg_out32(s, p->vset_insn);
744	}
745
746	static MemOp set_vtype_len(TCGContext *s, TCGType type)
747	{
748	if (type != s->riscv_cur_type) {
749	set_vtype(s, type, MO_64);
750	}
751	return s->riscv_cur_vsew;
752	}
753
754	static void set_vtype_len_sew(TCGContext *s, TCGType type, MemOp vsew)
755	{
756	if (type != s->riscv_cur_type \|\| vsew != s->riscv_cur_vsew) {
757	set_vtype(s, type, vsew);
758	}
759	}
760
761	/*
762	* TCG intrinsics
763	*/
764
765	static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
766	{
767	if (ret == arg) {
768	return true;
769	}
770	switch (type) {
771	case TCG_TYPE_I32:
772	case TCG_TYPE_I64:
773	tcg_out_opc_imm(s, OPC_ADDI, ret, arg, 0);
774	break;
775	case TCG_TYPE_V64:
776	case TCG_TYPE_V128:
777	case TCG_TYPE_V256:
778	{
779	int lmul = type - riscv_lg2_vlenb;
780	int nf = 1 << MAX(lmul, 0);
781	tcg_out_opc_vi(s, OPC_VMVNR_V, ret, arg, nf - 1);
782	}
783	break;
784	default:
785	g_assert_not_reached();
786	}
787	return true;
788	}
789
790	static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
791	tcg_target_long val)
792	{
793	tcg_target_long lo, hi, tmp;
794	int shift, ret;
795
796	if (type == TCG_TYPE_I32) {
797	val = (int32_t)val;
798	}
799
800	lo = sextreg(val, 0, 12);
801	if (val == lo) {
802	tcg_out_opc_imm(s, OPC_ADDI, rd, TCG_REG_ZERO, lo);
803	return;
804	}
805
806	hi = val - lo;
807	if (val == (int32_t)val) {
808	tcg_out_opc_upper(s, OPC_LUI, rd, hi);
809	if (lo != 0) {
810	tcg_out_opc_imm(s, OPC_ADDIW, rd, rd, lo);
811	}
812	return;
813	}
814
815	tmp = tcg_pcrel_diff(s, (void *)val);
816	if (tmp == (int32_t)tmp) {
817	tcg_out_opc_upper(s, OPC_AUIPC, rd, 0);
818	tcg_out_opc_imm(s, OPC_ADDI, rd, rd, 0);
819	ret = reloc_call(s->code_ptr - 2, (const tcg_insn_unit *)val);
820	tcg_debug_assert(ret == true);
821	return;
822	}
823
824	/* Look for a single 20-bit section. */
825	shift = ctz64(val);
826	tmp = val >> shift;
827	if (tmp == sextreg(tmp, 0, 20)) {
828	tcg_out_opc_upper(s, OPC_LUI, rd, tmp << 12);
829	if (shift > 12) {
830	tcg_out_opc_imm(s, OPC_SLLI, rd, rd, shift - 12);
831	} else {
832	tcg_out_opc_imm(s, OPC_SRAI, rd, rd, 12 - shift);
833	}
834	return;
835	}
836
837	/* Look for a few high zero bits, with lots of bits set in the middle. */
838	shift = clz64(val);
839	tmp = val << shift;
840	if (tmp == sextreg(tmp, 12, 20) << 12) {
841	tcg_out_opc_upper(s, OPC_LUI, rd, tmp);
842	tcg_out_opc_imm(s, OPC_SRLI, rd, rd, shift);
843	return;
844	} else if (tmp == sextreg(tmp, 0, 12)) {
845	tcg_out_opc_imm(s, OPC_ADDI, rd, TCG_REG_ZERO, tmp);
846	tcg_out_opc_imm(s, OPC_SRLI, rd, rd, shift);
847	return;
848	}
849
850	/* Drop into the constant pool. */
851	new_pool_label(s, val, R_RISCV_CALL, s->code_ptr, 0);
852	tcg_out_opc_upper(s, OPC_AUIPC, rd, 0);
853	tcg_out_opc_imm(s, OPC_LD, rd, rd, 0);
854	}
855
856	static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
857	{
858	return false;
859	}
860
861	static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
862	tcg_target_long imm)
863	{
864	/* This function is only used for passing structs by reference. */
865	g_assert_not_reached();
866	}
867
868	static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg)
869	{
870	tcg_out_opc_imm(s, OPC_ANDI, ret, arg, 0xff);
871	}
872
873	static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg)
874	{
875	if (cpuinfo & CPUINFO_ZBB) {
876	tcg_out_opc_reg(s, OPC_ZEXT_H, ret, arg, TCG_REG_ZERO);
877	} else {
878	tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16);
879	tcg_out_opc_imm(s, OPC_SRLIW, ret, ret, 16);
880	}
881	}
882
883	static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg)
884	{
885	if (cpuinfo & CPUINFO_ZBA) {
886	tcg_out_opc_reg(s, OPC_ADD_UW, ret, arg, TCG_REG_ZERO);
887	} else {
888	tcg_out_opc_imm(s, OPC_SLLI, ret, arg, 32);
889	tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32);
890	}
891	}
892
893	static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
894	{
895	if (cpuinfo & CPUINFO_ZBB) {
896	tcg_out_opc_imm(s, OPC_SEXT_B, ret, arg, 0);
897	} else {
898	tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24);
899	tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24);
900	}
901	}
902
903	static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
904	{
905	if (cpuinfo & CPUINFO_ZBB) {
906	tcg_out_opc_imm(s, OPC_SEXT_H, ret, arg, 0);
907	} else {
908	tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16);
909	tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16);
910	}
911	}
912
913	static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg)
914	{
915	tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0);
916	}
917
918	static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
919	{
920	if (ret != arg) {
921	tcg_out_ext32s(s, ret, arg);
922	}
923	}
924
925	static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
926	{
927	tcg_out_ext32u(s, ret, arg);
928	}
929
930	static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg)
931	{
932	tcg_out_ext32s(s, ret, arg);
933	}
934
935	static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data,
936	TCGReg addr, intptr_t offset)
937	{
938	intptr_t imm12 = sextreg(offset, 0, 12);
939
940	if (offset != imm12) {
941	intptr_t diff = tcg_pcrel_diff(s, (void *)offset);
942
943	if (addr == TCG_REG_ZERO && diff == (int32_t)diff) {
944	imm12 = sextreg(diff, 0, 12);
945	tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP2, diff - imm12);
946	} else {
947	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP2, offset - imm12);
948	if (addr != TCG_REG_ZERO) {
949	tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, addr);
950	}
951	}
952	addr = TCG_REG_TMP2;
953	}
954
955	switch (opc) {
956	case OPC_SB:
957	case OPC_SH:
958	case OPC_SW:
959	case OPC_SD:
960	tcg_out_opc_store(s, opc, addr, data, imm12);
961	break;
962	case OPC_LB:
963	case OPC_LBU:
964	case OPC_LH:
965	case OPC_LHU:
966	case OPC_LW:
967	case OPC_LWU:
968	case OPC_LD:
969	tcg_out_opc_imm(s, opc, data, addr, imm12);
970	break;
971	default:
972	g_assert_not_reached();
973	}
974	}
975
976	static void tcg_out_vec_ldst(TCGContext *s, RISCVInsn opc, TCGReg data,
977	TCGReg addr, intptr_t offset)
978	{
979	tcg_debug_assert(data >= TCG_REG_V0);
980	tcg_debug_assert(addr < TCG_REG_V0);
981
982	if (offset) {
983	tcg_debug_assert(addr != TCG_REG_ZERO);
984	if (offset == sextreg(offset, 0, 12)) {
985	tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_TMP0, addr, offset);
986	} else {
987	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, offset);
988	tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP0, addr);
989	}
990	addr = TCG_REG_TMP0;
991	}
992	tcg_out32(s, encode_v(opc, data, addr, 0, true));
993	}
994
995	static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
996	TCGReg arg1, intptr_t arg2)
997	{
998	RISCVInsn insn;
999
1000	switch (type) {
1001	case TCG_TYPE_I32:
1002	tcg_out_ldst(s, OPC_LW, arg, arg1, arg2);
1003	break;
1004	case TCG_TYPE_I64:
1005	tcg_out_ldst(s, OPC_LD, arg, arg1, arg2);
1006	break;
1007	case TCG_TYPE_V64:
1008	case TCG_TYPE_V128:
1009	case TCG_TYPE_V256:
1010	if (type >= riscv_lg2_vlenb) {
1011	static const RISCVInsn whole_reg_ld[] = {
1012	OPC_VL1RE64_V, OPC_VL2RE64_V, OPC_VL4RE64_V, OPC_VL8RE64_V
1013	};
1014	unsigned idx = type - riscv_lg2_vlenb;
1015
1016	tcg_debug_assert(idx < ARRAY_SIZE(whole_reg_ld));
1017	insn = whole_reg_ld[idx];
1018	} else {
1019	static const RISCVInsn unit_stride_ld[] = {
1020	OPC_VLE8_V, OPC_VLE16_V, OPC_VLE32_V, OPC_VLE64_V
1021	};
1022	MemOp prev_vsew = set_vtype_len(s, type);
1023
1024	tcg_debug_assert(prev_vsew < ARRAY_SIZE(unit_stride_ld));
1025	insn = unit_stride_ld[prev_vsew];
1026	}
1027	tcg_out_vec_ldst(s, insn, arg, arg1, arg2);
1028	break;
1029	default:
1030	g_assert_not_reached();
1031	}
1032	}
1033
1034	static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
1035	TCGReg arg1, intptr_t arg2)
1036	{
1037	RISCVInsn insn;
1038
1039	switch (type) {
1040	case TCG_TYPE_I32:
1041	tcg_out_ldst(s, OPC_SW, arg, arg1, arg2);
1042	break;
1043	case TCG_TYPE_I64:
1044	tcg_out_ldst(s, OPC_SD, arg, arg1, arg2);
1045	break;
1046	case TCG_TYPE_V64:
1047	case TCG_TYPE_V128:
1048	case TCG_TYPE_V256:
1049	if (type >= riscv_lg2_vlenb) {
1050	static const RISCVInsn whole_reg_st[] = {
1051	OPC_VS1R_V, OPC_VS2R_V, OPC_VS4R_V, OPC_VS8R_V
1052	};
1053	unsigned idx = type - riscv_lg2_vlenb;
1054
1055	tcg_debug_assert(idx < ARRAY_SIZE(whole_reg_st));
1056	insn = whole_reg_st[idx];
1057	} else {
1058	static const RISCVInsn unit_stride_st[] = {
1059	OPC_VSE8_V, OPC_VSE16_V, OPC_VSE32_V, OPC_VSE64_V
1060	};
1061	MemOp prev_vsew = set_vtype_len(s, type);
1062
1063	tcg_debug_assert(prev_vsew < ARRAY_SIZE(unit_stride_st));
1064	insn = unit_stride_st[prev_vsew];
1065	}
1066	tcg_out_vec_ldst(s, insn, arg, arg1, arg2);
1067	break;
1068	default:
1069	g_assert_not_reached();
1070	}
1071	}
1072
1073	static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1074	TCGReg base, intptr_t ofs)
1075	{
1076	if (val == 0) {
1077	tcg_out_st(s, type, TCG_REG_ZERO, base, ofs);
1078	return true;
1079	}
1080	return false;
1081	}
1082
1083	static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
1084	TCGReg dst, TCGReg src)
1085	{
1086	set_vtype_len_sew(s, type, vece);
1087	tcg_out_opc_vx(s, OPC_VMV_V_X, dst, 0, src);
1088	return true;
1089	}
1090
1091	static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
1092	TCGReg dst, TCGReg base, intptr_t offset)
1093	{
1094	/* Note set_vtype* may clobber TMP0, so do that first. */
1095	set_vtype_len_sew(s, type, vece);
1096	tcg_out_ld(s, TCG_TYPE_REG, TCG_REG_TMP0, base, offset);
1097	tcg_out_opc_vx(s, OPC_VMV_V_X, dst, 0, TCG_REG_TMP0);
1098	return true;
1099	}
1100
1101	static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
1102	TCGReg dst, int64_t arg)
1103	{
1104	/* Arg is replicated by VECE; extract the highest element. */
1105	arg >>= (-8 << vece) & 63;
1106
1107	/* Note set_vtype* may clobber TMP0, so do that first. */
1108	if (arg == 0 \|\| arg == -1) {
1109	set_vtype_len(s, type);
1110	} else {
1111	set_vtype_len_sew(s, type, vece);
1112	}
1113
1114	if (arg >= -16 && arg < 16) {
1115	tcg_out_opc_vi(s, OPC_VMV_V_I, dst, 0, arg);
1116	} else {
1117	tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP0, arg);
1118	tcg_out_opc_vx(s, OPC_VMV_V_X, dst, 0, TCG_REG_TMP0);
1119	}
1120	}
1121
1122	static void tcg_out_br(TCGContext s, TCGLabel l)
1123	{
1124	tcg_out_reloc(s, s->code_ptr, R_RISCV_JAL, l, 0);
1125	tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0);
1126	}
1127
1128	static const struct {
1129	RISCVInsn op;
1130	bool swap;
1131	} tcg_brcond_to_riscv[] = {
1132	[TCG_COND_EQ] = { OPC_BEQ, false },
1133	[TCG_COND_NE] = { OPC_BNE, false },
1134	[TCG_COND_LT] = { OPC_BLT, false },
1135	[TCG_COND_GE] = { OPC_BGE, false },
1136	[TCG_COND_LE] = { OPC_BGE, true },
1137	[TCG_COND_GT] = { OPC_BLT, true },
1138	[TCG_COND_LTU] = { OPC_BLTU, false },
1139	[TCG_COND_GEU] = { OPC_BGEU, false },
1140	[TCG_COND_LEU] = { OPC_BGEU, true },
1141	[TCG_COND_GTU] = { OPC_BLTU, true }
1142	};
1143
1144	static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond,
1145	TCGReg arg1, TCGReg arg2, TCGLabel *l)
1146	{
1147	RISCVInsn op = tcg_brcond_to_riscv[cond].op;
1148
1149	tcg_debug_assert(op != 0);
1150
1151	if (tcg_brcond_to_riscv[cond].swap) {
1152	TCGReg t = arg1;
1153	arg1 = arg2;
1154	arg2 = t;
1155	}
1156
1157	tcg_out_reloc(s, s->code_ptr, R_RISCV_BRANCH, l, 0);
1158	tcg_out_opc_branch(s, op, arg1, arg2, 0);
1159	}
1160
1161	static const TCGOutOpBrcond outop_brcond = {
1162	.base.static_constraint = C_O0_I2(r, rz),
1163	.out_rr = tgen_brcond,
1164	};
1165
1166	#define SETCOND_INV TCG_TARGET_NB_REGS
1167	#define SETCOND_NEZ (SETCOND_INV << 1)
1168	#define SETCOND_FLAGS (SETCOND_INV \| SETCOND_NEZ)
1169
1170	static int tcg_out_setcond_int(TCGContext *s, TCGCond cond, TCGReg ret,
1171	TCGReg arg1, tcg_target_long arg2, bool c2)
1172	{
1173	int flags = 0;
1174
1175	switch (cond) {
1176	case TCG_COND_EQ: /* -> NE */
1177	case TCG_COND_GE: /* -> LT */
1178	case TCG_COND_GEU: /* -> LTU */
1179	case TCG_COND_GT: /* -> LE */
1180	case TCG_COND_GTU: /* -> LEU */
1181	cond = tcg_invert_cond(cond);
1182	flags ^= SETCOND_INV;
1183	break;
1184	default:
1185	break;
1186	}
1187
1188	switch (cond) {
1189	case TCG_COND_LE:
1190	case TCG_COND_LEU:
1191	/*
1192	* If we have a constant input, the most efficient way to implement
1193	* LE is by adding 1 and using LT. Watch out for wrap around for LEU.
1194	* We don't need to care for this for LE because the constant input
1195	* is constrained to signed 12-bit, and 0x800 is representable in the
1196	* temporary register.
1197	*/
1198	if (c2) {
1199	if (cond == TCG_COND_LEU) {
1200	/* unsigned <= -1 is true */
1201	if (arg2 == -1) {
1202	tcg_out_movi(s, TCG_TYPE_REG, ret, !(flags & SETCOND_INV));
1203	return ret;
1204	}
1205	cond = TCG_COND_LTU;
1206	} else {
1207	cond = TCG_COND_LT;
1208	}
1209	tcg_debug_assert(arg2 <= 0x7ff);
1210	if (++arg2 == 0x800) {
1211	tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP0, arg2);
1212	arg2 = TCG_REG_TMP0;
1213	c2 = false;
1214	}
1215	} else {
1216	TCGReg tmp = arg2;
1217	arg2 = arg1;
1218	arg1 = tmp;
1219	cond = tcg_swap_cond(cond); /* LE -> GE */
1220	cond = tcg_invert_cond(cond); /* GE -> LT */
1221	flags ^= SETCOND_INV;
1222	}
1223	break;
1224	default:
1225	break;
1226	}
1227
1228	switch (cond) {
1229	case TCG_COND_NE:
1230	flags \|= SETCOND_NEZ;
1231	if (!c2) {
1232	tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
1233	} else if (arg2 == 0) {
1234	ret = arg1;
1235	} else {
1236	tcg_out_opc_imm(s, OPC_XORI, ret, arg1, arg2);
1237	}
1238	break;
1239
1240	case TCG_COND_LT:
1241	if (c2) {
1242	tcg_out_opc_imm(s, OPC_SLTI, ret, arg1, arg2);
1243	} else {
1244	tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2);
1245	}
1246	break;
1247
1248	case TCG_COND_LTU:
1249	if (c2) {
1250	tcg_out_opc_imm(s, OPC_SLTIU, ret, arg1, arg2);
1251	} else {
1252	tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2);
1253	}
1254	break;
1255
1256	default:
1257	g_assert_not_reached();
1258	}
1259
1260	return ret \| flags;
1261	}
1262
1263	static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
1264	TCGReg arg1, tcg_target_long arg2, bool c2)
1265	{
1266	int tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2, c2);
1267
1268	if (tmpflags != ret) {
1269	TCGReg tmp = tmpflags & ~SETCOND_FLAGS;
1270
1271	switch (tmpflags & SETCOND_FLAGS) {
1272	case SETCOND_INV:
1273	/* Intermediate result is boolean: simply invert. */
1274	tcg_out_opc_imm(s, OPC_XORI, ret, tmp, 1);
1275	break;
1276	case SETCOND_NEZ:
1277	/* Intermediate result is zero/non-zero: test != 0. */
1278	tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, tmp);
1279	break;
1280	case SETCOND_NEZ \| SETCOND_INV:
1281	/* Intermediate result is zero/non-zero: test == 0. */
1282	tcg_out_opc_imm(s, OPC_SLTIU, ret, tmp, 1);
1283	break;
1284	default:
1285	g_assert_not_reached();
1286	}
1287	}
1288	}
1289
1290	static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
1291	TCGReg dest, TCGReg arg1, TCGReg arg2)
1292	{
1293	tcg_out_setcond(s, cond, dest, arg1, arg2, false);
1294	}
1295
1296	static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond,
1297	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
1298	{
1299	tcg_out_setcond(s, cond, dest, arg1, arg2, true);
1300	}
1301
1302	static const TCGOutOpSetcond outop_setcond = {
1303	.base.static_constraint = C_O1_I2(r, r, rI),
1304	.out_rrr = tgen_setcond,
1305	.out_rri = tgen_setcondi,
1306	};
1307
1308	static void tcg_out_negsetcond(TCGContext *s, TCGCond cond, TCGReg ret,
1309	TCGReg arg1, tcg_target_long arg2, bool c2)
1310	{
1311	int tmpflags;
1312	TCGReg tmp;
1313
1314	/* For LT/GE comparison against 0, replicate the sign bit. */
1315	if (c2 && arg2 == 0) {
1316	switch (cond) {
1317	case TCG_COND_GE:
1318	tcg_out_opc_imm(s, OPC_XORI, ret, arg1, -1);
1319	arg1 = ret;
1320	/* fall through */
1321	case TCG_COND_LT:
1322	tcg_out_opc_imm(s, OPC_SRAI, ret, arg1, TCG_TARGET_REG_BITS - 1);
1323	return;
1324	default:
1325	break;
1326	}
1327	}
1328
1329	tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2, c2);
1330	tmp = tmpflags & ~SETCOND_FLAGS;
1331
1332	/* If intermediate result is zero/non-zero: test != 0. */
1333	if (tmpflags & SETCOND_NEZ) {
1334	tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, tmp);
1335	tmp = ret;
1336	}
1337
1338	/* Produce the 0/-1 result. */
1339	if (tmpflags & SETCOND_INV) {
1340	tcg_out_opc_imm(s, OPC_ADDI, ret, tmp, -1);
1341	} else {
1342	tcg_out_opc_reg(s, OPC_SUB, ret, TCG_REG_ZERO, tmp);
1343	}
1344	}
1345
1346	static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond,
1347	TCGReg dest, TCGReg arg1, TCGReg arg2)
1348	{
1349	tcg_out_negsetcond(s, cond, dest, arg1, arg2, false);
1350	}
1351
1352	static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond,
1353	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
1354	{
1355	tcg_out_negsetcond(s, cond, dest, arg1, arg2, true);
1356	}
1357
1358	static const TCGOutOpSetcond outop_negsetcond = {
1359	.base.static_constraint = C_O1_I2(r, r, rI),
1360	.out_rrr = tgen_negsetcond,
1361	.out_rri = tgen_negsetcondi,
1362	};
1363
1364	static void tcg_out_movcond_zicond(TCGContext *s, TCGReg ret, TCGReg test_ne,
1365	int val1, bool c_val1,
1366	int val2, bool c_val2)
1367	{
1368	if (val1 == 0) {
1369	if (c_val2) {
1370	tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val2);
1371	val2 = TCG_REG_TMP1;
1372	}
1373	tcg_out_opc_reg(s, OPC_CZERO_NEZ, ret, val2, test_ne);
1374	return;
1375	}
1376
1377	if (val2 == 0) {
1378	if (c_val1) {
1379	tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val1);
1380	val1 = TCG_REG_TMP1;
1381	}
1382	tcg_out_opc_reg(s, OPC_CZERO_EQZ, ret, val1, test_ne);
1383	return;
1384	}
1385
1386	if (c_val2) {
1387	if (c_val1) {
1388	tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val1 - val2);
1389	} else {
1390	tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_TMP1, val1, -val2);
1391	}
1392	tcg_out_opc_reg(s, OPC_CZERO_EQZ, ret, TCG_REG_TMP1, test_ne);
1393	tcg_out_opc_imm(s, OPC_ADDI, ret, ret, val2);
1394	return;
1395	}
1396
1397	if (c_val1) {
1398	tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_TMP1, val2, -val1);
1399	tcg_out_opc_reg(s, OPC_CZERO_NEZ, ret, TCG_REG_TMP1, test_ne);
1400	tcg_out_opc_imm(s, OPC_ADDI, ret, ret, val1);
1401	return;
1402	}
1403
1404	tcg_out_opc_reg(s, OPC_CZERO_NEZ, TCG_REG_TMP1, val2, test_ne);
1405	tcg_out_opc_reg(s, OPC_CZERO_EQZ, TCG_REG_TMP0, val1, test_ne);
1406	tcg_out_opc_reg(s, OPC_OR, ret, TCG_REG_TMP0, TCG_REG_TMP1);
1407	}
1408
1409	static void tcg_out_movcond_br1(TCGContext *s, TCGCond cond, TCGReg ret,
1410	TCGReg cmp1, TCGReg cmp2,
1411	int val, bool c_val)
1412	{
1413	RISCVInsn op;
1414	int disp = 8;
1415
1416	tcg_debug_assert((unsigned)cond < ARRAY_SIZE(tcg_brcond_to_riscv));
1417	op = tcg_brcond_to_riscv[cond].op;
1418	tcg_debug_assert(op != 0);
1419
1420	if (tcg_brcond_to_riscv[cond].swap) {
1421	tcg_out_opc_branch(s, op, cmp2, cmp1, disp);
1422	} else {
1423	tcg_out_opc_branch(s, op, cmp1, cmp2, disp);
1424	}
1425	if (c_val) {
1426	tcg_out_opc_imm(s, OPC_ADDI, ret, TCG_REG_ZERO, val);
1427	} else {
1428	tcg_out_opc_imm(s, OPC_ADDI, ret, val, 0);
1429	}
1430	}
1431
1432	static void tcg_out_movcond_br2(TCGContext *s, TCGCond cond, TCGReg ret,
1433	TCGReg cmp1, TCGReg cmp2,
1434	int val1, bool c_val1,
1435	int val2, bool c_val2)
1436	{
1437	TCGReg tmp;
1438
1439	/* TCG optimizer reorders to prefer ret matching val2. */
1440	if (!c_val2 && ret == val2) {
1441	cond = tcg_invert_cond(cond);
1442	tcg_out_movcond_br1(s, cond, ret, cmp1, cmp2, val1, c_val1);
1443	return;
1444	}
1445
1446	if (!c_val1 && ret == val1) {
1447	tcg_out_movcond_br1(s, cond, ret, cmp1, cmp2, val2, c_val2);
1448	return;
1449	}
1450
1451	tmp = (ret == cmp1 \|\| ret == cmp2 ? TCG_REG_TMP1 : ret);
1452	if (c_val1) {
1453	tcg_out_movi(s, TCG_TYPE_REG, tmp, val1);
1454	} else {
1455	tcg_out_mov(s, TCG_TYPE_REG, tmp, val1);
1456	}
1457	tcg_out_movcond_br1(s, cond, tmp, cmp1, cmp2, val2, c_val2);
1458	tcg_out_mov(s, TCG_TYPE_REG, ret, tmp);
1459	}
1460
1461	static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond,
1462	TCGReg ret, TCGReg cmp1, TCGArg cmp2, bool c_cmp2,
1463	TCGArg val1, bool c_val1,
1464	TCGArg val2, bool c_val2)
1465	{
1466	int tmpflags;
1467	TCGReg t;
1468
1469	if (!(cpuinfo & CPUINFO_ZICOND) && (!c_cmp2 \|\| cmp2 == 0)) {
1470	tcg_out_movcond_br2(s, cond, ret, cmp1, cmp2,
1471	val1, c_val1, val2, c_val2);
1472	return;
1473	}
1474
1475	tmpflags = tcg_out_setcond_int(s, cond, TCG_REG_TMP0, cmp1, cmp2, c_cmp2);
1476	t = tmpflags & ~SETCOND_FLAGS;
1477
1478	if (cpuinfo & CPUINFO_ZICOND) {
1479	if (tmpflags & SETCOND_INV) {
1480	tcg_out_movcond_zicond(s, ret, t, val2, c_val2, val1, c_val1);
1481	} else {
1482	tcg_out_movcond_zicond(s, ret, t, val1, c_val1, val2, c_val2);
1483	}
1484	} else {
1485	cond = tmpflags & SETCOND_INV ? TCG_COND_EQ : TCG_COND_NE;
1486	tcg_out_movcond_br2(s, cond, ret, t, TCG_REG_ZERO,
1487	val1, c_val1, val2, c_val2);
1488	}
1489	}
1490
1491	static const TCGOutOpMovcond outop_movcond = {
1492	.base.static_constraint = C_O1_I4(r, r, rI, rM, rM),
1493	.out = tcg_out_movcond,
1494	};
1495
1496	static void tcg_out_cltz(TCGContext *s, TCGType type, RISCVInsn insn,
1497	TCGReg ret, TCGReg src1, int src2, bool c_src2)
1498	{
1499	tcg_out_opc_imm(s, insn, ret, src1, 0);
1500
1501	if (!c_src2 \|\| src2 != (type == TCG_TYPE_I32 ? 32 : 64)) {
1502	/*
1503	* The requested zero result does not match the insn, so adjust.
1504	* Note that constraints put 'ret' in a new register, so the
1505	* computation above did not clobber either 'src1' or 'src2'.
1506	*/
1507	tcg_out_movcond(s, type, TCG_COND_EQ, ret, src1, 0, true,
1508	src2, c_src2, ret, false);
1509	}
1510	}
1511
1512	static void tcg_out_cmpsel(TCGContext *s, TCGType type, unsigned vece,
1513	TCGCond cond, TCGReg ret,
1514	TCGReg cmp1, TCGReg cmp2, bool c_cmp2,
1515	TCGReg val1, bool c_val1,
1516	TCGReg val2, bool c_val2)
1517	{
1518	set_vtype_len_sew(s, type, vece);
1519
1520	/* Use only vmerge_vim if possible, by inverting the test. */
1521	if (c_val2 && !c_val1) {
1522	TCGArg temp = val1;
1523	cond = tcg_invert_cond(cond);
1524	val1 = val2;
1525	val2 = temp;
1526	c_val1 = true;
1527	c_val2 = false;
1528	}
1529
1530	/* Perform the comparison into V0 mask. */
1531	if (c_cmp2) {
1532	tcg_out_opc_vi(s, tcg_cmpcond_to_rvv_vi[cond].op, TCG_REG_V0, cmp1,
1533	cmp2 - tcg_cmpcond_to_rvv_vi[cond].adjust);
1534	} else if (tcg_cmpcond_to_rvv_vv[cond].swap) {
1535	tcg_out_opc_vv(s, tcg_cmpcond_to_rvv_vv[cond].op,
1536	TCG_REG_V0, cmp2, cmp1);
1537	} else {
1538	tcg_out_opc_vv(s, tcg_cmpcond_to_rvv_vv[cond].op,
1539	TCG_REG_V0, cmp1, cmp2);
1540	}
1541	if (c_val1) {
1542	if (c_val2) {
1543	tcg_out_opc_vi(s, OPC_VMV_V_I, ret, 0, val2);
1544	val2 = ret;
1545	}
1546	/* vd[i] == v0.mask[i] ? imm : vs2[i] */
1547	tcg_out_opc_vim_mask(s, OPC_VMERGE_VIM, ret, val2, val1);
1548	} else {
1549	/* vd[i] == v0.mask[i] ? vs1[i] : vs2[i] */
1550	tcg_out_opc_vvm_mask(s, OPC_VMERGE_VVM, ret, val2, val1);
1551	}
1552	}
1553
1554	static void tcg_out_vshifti(TCGContext *s, RISCVInsn opc_vi, RISCVInsn opc_vx,
1555	TCGReg dst, TCGReg src, unsigned imm)
1556	{
1557	if (imm < 32) {
1558	tcg_out_opc_vi(s, opc_vi, dst, src, imm);
1559	} else {
1560	tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_TMP0, imm);
1561	tcg_out_opc_vx(s, opc_vx, dst, src, TCG_REG_TMP0);
1562	}
1563	}
1564
1565	static void init_setting_vtype(TCGContext *s)
1566	{
1567	s->riscv_cur_type = TCG_TYPE_COUNT;
1568	}
1569
1570	static void tcg_out_call_int(TCGContext s, const tcg_insn_unit arg, bool tail)
1571	{
1572	TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA;
1573	ptrdiff_t offset = tcg_pcrel_diff(s, arg);
1574	int ret;
1575
1576	init_setting_vtype(s);
1577
1578	tcg_debug_assert((offset & 1) == 0);
1579	if (offset == sextreg(offset, 0, 20)) {
1580	/* short jump: -2097150 to 2097152 */
1581	tcg_out_opc_jump(s, OPC_JAL, link, offset);
1582	} else if (offset == (int32_t)offset) {
1583	/* long jump: -2147483646 to 2147483648 */
1584	tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0);
1585	tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0);
1586	ret = reloc_call(s->code_ptr - 2, arg);
1587	tcg_debug_assert(ret == true);
1588	} else {
1589	/* far jump: 64-bit */
1590	tcg_target_long imm = sextreg((tcg_target_long)arg, 0, 12);
1591	tcg_target_long base = (tcg_target_long)arg - imm;
1592	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, base);
1593	tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, imm);
1594	}
1595	}
1596
1597	static void tcg_out_call(TCGContext s, const tcg_insn_unit arg,
1598	const TCGHelperInfo *info)
1599	{
1600	tcg_out_call_int(s, arg, false);
1601	}
1602
1603	static void tcg_out_mb(TCGContext *s, unsigned a0)
1604	{
1605	tcg_insn_unit insn = OPC_FENCE;
1606
1607	if (a0 & TCG_MO_LD_LD) {
1608	insn \|= 0x02200000;
1609	}
1610	if (a0 & TCG_MO_ST_LD) {
1611	insn \|= 0x01200000;
1612	}
1613	if (a0 & TCG_MO_LD_ST) {
1614	insn \|= 0x02100000;
1615	}
1616	if (a0 & TCG_MO_ST_ST) {
1617	insn \|= 0x01100000;
1618	}
1619	tcg_out32(s, insn);
1620	}
1621
1622	/*
1623	* Load/store and TLB
1624	*/
1625
1626	static void tcg_out_goto(TCGContext s, const tcg_insn_unit target)
1627	{
1628	tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0);
1629	bool ok = reloc_jimm20(s->code_ptr - 1, target);
1630	tcg_debug_assert(ok);
1631	}
1632
1633	bool tcg_target_has_memory_bswap(MemOp memop)
1634	{
1635	return false;
1636	}
1637
1638	/* We have three temps, we might as well expose them. */
1639	static const TCGLdstHelperParam ldst_helper_param = {
1640	.ntmp = 3, .tmp = { TCG_REG_TMP0, TCG_REG_TMP1, TCG_REG_TMP2 }
1641	};
1642
1643	static bool tcg_out_qemu_ld_slow_path(TCGContext s, TCGLabelQemuLdst l)
1644	{
1645	MemOp opc = get_memop(l->oi);
1646
1647	/* resolve label address */
1648	if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
1649	return false;
1650	}
1651
1652	/* call load helper */
1653	tcg_out_ld_helper_args(s, l, &ldst_helper_param);
1654	tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SSIZE], false);
1655	tcg_out_ld_helper_ret(s, l, true, &ldst_helper_param);
1656
1657	tcg_out_goto(s, l->raddr);
1658	return true;
1659	}
1660
1661	static bool tcg_out_qemu_st_slow_path(TCGContext s, TCGLabelQemuLdst l)
1662	{
1663	MemOp opc = get_memop(l->oi);
1664
1665	/* resolve label address */
1666	if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
1667	return false;
1668	}
1669
1670	/* call store helper */
1671	tcg_out_st_helper_args(s, l, &ldst_helper_param);
1672	tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE], false);
1673
1674	tcg_out_goto(s, l->raddr);
1675	return true;
1676	}
1677
1678	/* We expect to use a 12-bit negative offset from ENV. */
1679	#define MIN_TLB_MASK_TABLE_OFS -(1 << 11)
1680
1681	/*
1682	* For system-mode, perform the TLB load and compare.
1683	* For user-mode, perform any required alignment tests.
1684	* In both cases, return a TCGLabelQemuLdst structure if the slow path
1685	* is required and fill in @h with the host address for the fast path.
1686	*/
1687	static TCGLabelQemuLdst prepare_host_addr(TCGContext s, TCGReg *pbase,
1688	TCGReg addr_reg, MemOpIdx oi,
1689	bool is_ld)
1690	{
1691	TCGType addr_type = s->addr_type;
1692	TCGLabelQemuLdst *ldst = NULL;
1693	MemOp opc = get_memop(oi);
1694	TCGAtomAlign aa;
1695	unsigned a_mask;
1696
1697	aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false);
1698	a_mask = (1u << aa.align) - 1;
1699
1700	if (tcg_use_softmmu) {
1701	unsigned s_bits = opc & MO_SIZE;
1702	unsigned s_mask = (1u << s_bits) - 1;
1703	int mem_index = get_mmuidx(oi);
1704	int fast_ofs = tlb_mask_table_ofs(s, mem_index);
1705	int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask);
1706	int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table);
1707	int compare_mask;
1708	TCGReg addr_adj;
1709
1710	ldst = new_ldst_label(s);
1711	ldst->is_ld = is_ld;
1712	ldst->oi = oi;
1713	ldst->addr_reg = addr_reg;
1714
1715	init_setting_vtype(s);
1716
1717	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_AREG0, mask_ofs);
1718	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, table_ofs);
1719
1720	tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addr_reg,
1721	TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1722	tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0);
1723	tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1);
1724
1725	/*
1726	* For aligned accesses, we check the first byte and include the
1727	* alignment bits within the address. For unaligned access, we
1728	* check that we don't cross pages using the address of the last
1729	* byte of the access.
1730	*/
1731	addr_adj = addr_reg;
1732	if (a_mask < s_mask) {
1733	addr_adj = TCG_REG_TMP0;
1734	tcg_out_opc_imm(s, addr_type == TCG_TYPE_I32 ? OPC_ADDIW : OPC_ADDI,
1735	addr_adj, addr_reg, s_mask - a_mask);
1736	}
1737	compare_mask = TARGET_PAGE_MASK \| a_mask;
1738	if (compare_mask == sextreg(compare_mask, 0, 12)) {
1739	tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_adj, compare_mask);
1740	} else {
1741	tcg_out_movi(s, addr_type, TCG_REG_TMP1, compare_mask);
1742	tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr_adj);
1743	}
1744
1745	/* Load the tlb comparator and the addend. */
1746	QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN);
1747	tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2,
1748	is_ld ? offsetof(CPUTLBEntry, addr_read)
1749	: offsetof(CPUTLBEntry, addr_write));
1750	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2,
1751	offsetof(CPUTLBEntry, addend));
1752
1753	/* Compare masked address with the TLB entry. */
1754	ldst->label_ptr[0] = s->code_ptr;
1755	tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0);
1756
1757	/* TLB Hit - translate address using addend. */
1758	if (addr_type != TCG_TYPE_I32) {
1759	tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, addr_reg, TCG_REG_TMP2);
1760	} else if (cpuinfo & CPUINFO_ZBA) {
1761	tcg_out_opc_reg(s, OPC_ADD_UW, TCG_REG_TMP0,
1762	addr_reg, TCG_REG_TMP2);
1763	} else {
1764	tcg_out_ext32u(s, TCG_REG_TMP0, addr_reg);
1765	tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0,
1766	TCG_REG_TMP0, TCG_REG_TMP2);
1767	}
1768	*pbase = TCG_REG_TMP0;
1769	} else {
1770	TCGReg base;
1771
1772	if (a_mask) {
1773	ldst = new_ldst_label(s);
1774	ldst->is_ld = is_ld;
1775	ldst->oi = oi;
1776	ldst->addr_reg = addr_reg;
1777
1778	init_setting_vtype(s);
1779
1780	/* We are expecting alignment max 7, so we can always use andi. */
1781	tcg_debug_assert(a_mask == sextreg(a_mask, 0, 12));
1782	tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, a_mask);
1783
1784	ldst->label_ptr[0] = s->code_ptr;
1785	tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP1, TCG_REG_ZERO, 0);
1786	}
1787
1788	if (guest_base != 0) {
1789	base = TCG_REG_TMP0;
1790	if (addr_type != TCG_TYPE_I32) {
1791	tcg_out_opc_reg(s, OPC_ADD, base, addr_reg,
1792	TCG_GUEST_BASE_REG);
1793	} else if (cpuinfo & CPUINFO_ZBA) {
1794	tcg_out_opc_reg(s, OPC_ADD_UW, base, addr_reg,
1795	TCG_GUEST_BASE_REG);
1796	} else {
1797	tcg_out_ext32u(s, base, addr_reg);
1798	tcg_out_opc_reg(s, OPC_ADD, base, base, TCG_GUEST_BASE_REG);
1799	}
1800	} else if (addr_type != TCG_TYPE_I32) {
1801	base = addr_reg;
1802	} else {
1803	base = TCG_REG_TMP0;
1804	tcg_out_ext32u(s, base, addr_reg);
1805	}
1806	*pbase = base;
1807	}
1808
1809	return ldst;
1810	}
1811
1812	static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val,
1813	TCGReg base, MemOp opc, TCGType type)
1814	{
1815	/* Byte swapping is left to middle-end expansion. */
1816	tcg_debug_assert((opc & MO_BSWAP) == 0);
1817
1818	switch (opc & (MO_SSIZE)) {
1819	case MO_UB:
1820	tcg_out_opc_imm(s, OPC_LBU, val, base, 0);
1821	break;
1822	case MO_SB:
1823	tcg_out_opc_imm(s, OPC_LB, val, base, 0);
1824	break;
1825	case MO_UW:
1826	tcg_out_opc_imm(s, OPC_LHU, val, base, 0);
1827	break;
1828	case MO_SW:
1829	tcg_out_opc_imm(s, OPC_LH, val, base, 0);
1830	break;
1831	case MO_UL:
1832	if (type == TCG_TYPE_I64) {
1833	tcg_out_opc_imm(s, OPC_LWU, val, base, 0);
1834	break;
1835	}
1836	/* FALLTHRU */
1837	case MO_SL:
1838	tcg_out_opc_imm(s, OPC_LW, val, base, 0);
1839	break;
1840	case MO_UQ:
1841	tcg_out_opc_imm(s, OPC_LD, val, base, 0);
1842	break;
1843	default:
1844	g_assert_not_reached();
1845	}
1846	}
1847
1848	static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data_reg,
1849	TCGReg addr_reg, MemOpIdx oi)
1850	{
1851	TCGLabelQemuLdst *ldst;
1852	TCGReg base;
1853
1854	ldst = prepare_host_addr(s, &base, addr_reg, oi, true);
1855	tcg_out_qemu_ld_direct(s, data_reg, base, get_memop(oi), type);
1856
1857	if (ldst) {
1858	ldst->type = type;
1859	ldst->datalo_reg = data_reg;
1860	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1861	}
1862	}
1863
1864	static const TCGOutOpQemuLdSt outop_qemu_ld = {
1865	.base.static_constraint = C_O1_I1(r, r),
1866	.out = tgen_qemu_ld,
1867	};
1868
1869	static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = {
1870	.base.static_constraint = C_NotImplemented,
1871	};
1872
1873	static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val,
1874	TCGReg base, MemOp opc)
1875	{
1876	/* Byte swapping is left to middle-end expansion. */
1877	tcg_debug_assert((opc & MO_BSWAP) == 0);
1878
1879	switch (opc & (MO_SSIZE)) {
1880	case MO_8:
1881	tcg_out_opc_store(s, OPC_SB, base, val, 0);
1882	break;
1883	case MO_16:
1884	tcg_out_opc_store(s, OPC_SH, base, val, 0);
1885	break;
1886	case MO_32:
1887	tcg_out_opc_store(s, OPC_SW, base, val, 0);
1888	break;
1889	case MO_64:
1890	tcg_out_opc_store(s, OPC_SD, base, val, 0);
1891	break;
1892	default:
1893	g_assert_not_reached();
1894	}
1895	}
1896
1897	static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data_reg,
1898	TCGReg addr_reg, MemOpIdx oi)
1899	{
1900	TCGLabelQemuLdst *ldst;
1901	TCGReg base;
1902
1903	ldst = prepare_host_addr(s, &base, addr_reg, oi, false);
1904	tcg_out_qemu_st_direct(s, data_reg, base, get_memop(oi));
1905
1906	if (ldst) {
1907	ldst->type = type;
1908	ldst->datalo_reg = data_reg;
1909	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1910	}
1911	}
1912
1913	static const TCGOutOpQemuLdSt outop_qemu_st = {
1914	.base.static_constraint = C_O0_I2(rz, r),
1915	.out = tgen_qemu_st,
1916	};
1917
1918	static const TCGOutOpQemuLdSt2 outop_qemu_st2 = {
1919	.base.static_constraint = C_NotImplemented,
1920	};
1921
1922	static const tcg_insn_unit *tb_ret_addr;
1923
1924	static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
1925	{
1926	/* Reuse the zeroing that exists for goto_ptr. */
1927	if (a0 == 0) {
1928	tcg_out_call_int(s, tcg_code_gen_epilogue, true);
1929	} else {
1930	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
1931	tcg_out_call_int(s, tb_ret_addr, true);
1932	}
1933	}
1934
1935	static void tcg_out_goto_tb(TCGContext *s, int which)
1936	{
1937	/* Direct branch will be patched by tb_target_set_jmp_target. */
1938	set_jmp_insn_offset(s, which);
1939	tcg_out32(s, OPC_JAL);
1940
1941	/* When branch is out of range, fall through to indirect. */
1942	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
1943	get_jmp_target_addr(s, which));
1944	tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
1945	set_jmp_reset_offset(s, which);
1946	}
1947
1948	static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0)
1949	{
1950	tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0);
1951	}
1952
1953	void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
1954	uintptr_t jmp_rx, uintptr_t jmp_rw)
1955	{
1956	uintptr_t addr = tb->jmp_target_addr[n];
1957	ptrdiff_t offset = addr - jmp_rx;
1958	tcg_insn_unit insn;
1959
1960	/* Either directly branch, or fall through to indirect branch. */
1961	if (offset == sextreg(offset, 0, 20)) {
1962	insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset);
1963	} else {
1964	insn = OPC_NOP;
1965	}
1966	qatomic_set((uint32_t *)jmp_rw, insn);
1967	flush_idcache_range(jmp_rx, jmp_rw, 4);
1968	}
1969
1970
1971	static void tgen_add(TCGContext *s, TCGType type,
1972	TCGReg a0, TCGReg a1, TCGReg a2)
1973	{
1974	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_ADDW : OPC_ADD;
1975	tcg_out_opc_reg(s, insn, a0, a1, a2);
1976	}
1977
1978	static void tgen_addi(TCGContext *s, TCGType type,
1979	TCGReg a0, TCGReg a1, tcg_target_long a2)
1980	{
1981	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_ADDIW : OPC_ADDI;
1982	tcg_out_opc_imm(s, insn, a0, a1, a2);
1983	}
1984
1985	static const TCGOutOpBinary outop_add = {
1986	.base.static_constraint = C_O1_I2(r, r, rI),
1987	.out_rrr = tgen_add,
1988	.out_rri = tgen_addi,
1989	};
1990
1991	static const TCGOutOpBinary outop_addco = {
1992	.base.static_constraint = C_NotImplemented,
1993	};
1994
1995	static const TCGOutOpAddSubCarry outop_addci = {
1996	.base.static_constraint = C_NotImplemented,
1997	};
1998
1999	static const TCGOutOpBinary outop_addcio = {
2000	.base.static_constraint = C_NotImplemented,
2001	};
2002
2003	static void tcg_out_set_carry(TCGContext *s)
2004	{
2005	g_assert_not_reached();
2006	}
2007
2008	static void tgen_and(TCGContext *s, TCGType type,
2009	TCGReg a0, TCGReg a1, TCGReg a2)
2010	{
2011	tcg_out_opc_reg(s, OPC_AND, a0, a1, a2);
2012	}
2013
2014	static void tgen_andi(TCGContext *s, TCGType type,
2015	TCGReg a0, TCGReg a1, tcg_target_long a2)
2016	{
2017	tcg_out_opc_imm(s, OPC_ANDI, a0, a1, a2);
2018	}
2019
2020	static const TCGOutOpBinary outop_and = {
2021	.base.static_constraint = C_O1_I2(r, r, rI),
2022	.out_rrr = tgen_and,
2023	.out_rri = tgen_andi,
2024	};
2025
2026	static void tgen_andc(TCGContext *s, TCGType type,
2027	TCGReg a0, TCGReg a1, TCGReg a2)
2028	{
2029	tcg_out_opc_reg(s, OPC_ANDN, a0, a1, a2);
2030	}
2031
2032	static TCGConstraintSetIndex cset_zbb_rrr(TCGType type, unsigned flags)
2033	{
2034	return cpuinfo & CPUINFO_ZBB ? C_O1_I2(r, r, r) : C_NotImplemented;
2035	}
2036
2037	static const TCGOutOpBinary outop_andc = {
2038	.base.static_constraint = C_Dynamic,
2039	.base.dynamic_constraint = cset_zbb_rrr,
2040	.out_rrr = tgen_andc,
2041	};
2042
2043	static void tgen_clz(TCGContext *s, TCGType type,
2044	TCGReg a0, TCGReg a1, TCGReg a2)
2045	{
2046	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CLZW : OPC_CLZ;
2047	tcg_out_cltz(s, type, insn, a0, a1, a2, false);
2048	}
2049
2050	static void tgen_clzi(TCGContext *s, TCGType type,
2051	TCGReg a0, TCGReg a1, tcg_target_long a2)
2052	{
2053	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CLZW : OPC_CLZ;
2054	tcg_out_cltz(s, type, insn, a0, a1, a2, true);
2055	}
2056
2057	static TCGConstraintSetIndex cset_clzctz(TCGType type, unsigned flags)
2058	{
2059	return cpuinfo & CPUINFO_ZBB ? C_N1_I2(r, r, rM) : C_NotImplemented;
2060	}
2061
2062	static const TCGOutOpBinary outop_clz = {
2063	.base.static_constraint = C_Dynamic,
2064	.base.dynamic_constraint = cset_clzctz,
2065	.out_rrr = tgen_clz,
2066	.out_rri = tgen_clzi,
2067	};
2068
2069	static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2070	{
2071	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CPOPW : OPC_CPOP;
2072	tcg_out_opc_imm(s, insn, a0, a1, 0);
2073	}
2074
2075	static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags)
2076	{
2077	return cpuinfo & CPUINFO_ZBB ? C_O1_I1(r, r) : C_NotImplemented;
2078	}
2079
2080	static const TCGOutOpUnary outop_ctpop = {
2081	.base.static_constraint = C_Dynamic,
2082	.base.dynamic_constraint = cset_ctpop,
2083	.out_rr = tgen_ctpop,
2084	};
2085
2086	static void tgen_ctz(TCGContext *s, TCGType type,
2087	TCGReg a0, TCGReg a1, TCGReg a2)
2088	{
2089	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CTZW : OPC_CTZ;
2090	tcg_out_cltz(s, type, insn, a0, a1, a2, false);
2091	}
2092
2093	static void tgen_ctzi(TCGContext *s, TCGType type,
2094	TCGReg a0, TCGReg a1, tcg_target_long a2)
2095	{
2096	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_CTZW : OPC_CTZ;
2097	tcg_out_cltz(s, type, insn, a0, a1, a2, true);
2098	}
2099
2100	static const TCGOutOpBinary outop_ctz = {
2101	.base.static_constraint = C_Dynamic,
2102	.base.dynamic_constraint = cset_clzctz,
2103	.out_rrr = tgen_ctz,
2104	.out_rri = tgen_ctzi,
2105	};
2106
2107	static void tgen_divs(TCGContext *s, TCGType type,
2108	TCGReg a0, TCGReg a1, TCGReg a2)
2109	{
2110	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_DIVW : OPC_DIV;
2111	tcg_out_opc_reg(s, insn, a0, a1, a2);
2112	}
2113
2114	static const TCGOutOpBinary outop_divs = {
2115	.base.static_constraint = C_O1_I2(r, r, r),
2116	.out_rrr = tgen_divs,
2117	};
2118
2119	static const TCGOutOpDivRem outop_divs2 = {
2120	.base.static_constraint = C_NotImplemented,
2121	};
2122
2123	static void tgen_divu(TCGContext *s, TCGType type,
2124	TCGReg a0, TCGReg a1, TCGReg a2)
2125	{
2126	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_DIVUW : OPC_DIVU;
2127	tcg_out_opc_reg(s, insn, a0, a1, a2);
2128	}
2129
2130	static const TCGOutOpBinary outop_divu = {
2131	.base.static_constraint = C_O1_I2(r, r, r),
2132	.out_rrr = tgen_divu,
2133	};
2134
2135	static const TCGOutOpDivRem outop_divu2 = {
2136	.base.static_constraint = C_NotImplemented,
2137	};
2138
2139	static void tgen_eqv(TCGContext *s, TCGType type,
2140	TCGReg a0, TCGReg a1, TCGReg a2)
2141	{
2142	tcg_out_opc_reg(s, OPC_XNOR, a0, a1, a2);
2143	}
2144
2145	static const TCGOutOpBinary outop_eqv = {
2146	.base.static_constraint = C_Dynamic,
2147	.base.dynamic_constraint = cset_zbb_rrr,
2148	.out_rrr = tgen_eqv,
2149	};
2150
2151	static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
2152	{
2153	tcg_out_opc_imm(s, OPC_SRAI, a0, a1, 32);
2154	}
2155
2156	static const TCGOutOpUnary outop_extrh_i64_i32 = {
2157	.base.static_constraint = C_O1_I1(r, r),
2158	.out_rr = tgen_extrh_i64_i32,
2159	};
2160
2161	static void tgen_mul(TCGContext *s, TCGType type,
2162	TCGReg a0, TCGReg a1, TCGReg a2)
2163	{
2164	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_MULW : OPC_MUL;
2165	tcg_out_opc_reg(s, insn, a0, a1, a2);
2166	}
2167
2168	static const TCGOutOpBinary outop_mul = {
2169	.base.static_constraint = C_O1_I2(r, r, r),
2170	.out_rrr = tgen_mul,
2171	};
2172
2173	static const TCGOutOpMul2 outop_muls2 = {
2174	.base.static_constraint = C_NotImplemented,
2175	};
2176
2177	static TCGConstraintSetIndex cset_mulh(TCGType type, unsigned flags)
2178	{
2179	return type == TCG_TYPE_I32 ? C_NotImplemented : C_O1_I2(r, r, r);
2180	}
2181
2182	static void tgen_mulsh(TCGContext *s, TCGType type,
2183	TCGReg a0, TCGReg a1, TCGReg a2)
2184	{
2185	tcg_out_opc_reg(s, OPC_MULH, a0, a1, a2);
2186	}
2187
2188	static const TCGOutOpBinary outop_mulsh = {
2189	.base.static_constraint = C_Dynamic,
2190	.base.dynamic_constraint = cset_mulh,
2191	.out_rrr = tgen_mulsh,
2192	};
2193
2194	static const TCGOutOpMul2 outop_mulu2 = {
2195	.base.static_constraint = C_NotImplemented,
2196	};
2197
2198	static void tgen_muluh(TCGContext *s, TCGType type,
2199	TCGReg a0, TCGReg a1, TCGReg a2)
2200	{
2201	tcg_out_opc_reg(s, OPC_MULHU, a0, a1, a2);
2202	}
2203
2204	static const TCGOutOpBinary outop_muluh = {
2205	.base.static_constraint = C_Dynamic,
2206	.base.dynamic_constraint = cset_mulh,
2207	.out_rrr = tgen_muluh,
2208	};
2209
2210	static const TCGOutOpBinary outop_nand = {
2211	.base.static_constraint = C_NotImplemented,
2212	};
2213
2214	static const TCGOutOpBinary outop_nor = {
2215	.base.static_constraint = C_NotImplemented,
2216	};
2217
2218	static void tgen_or(TCGContext *s, TCGType type,
2219	TCGReg a0, TCGReg a1, TCGReg a2)
2220	{
2221	tcg_out_opc_reg(s, OPC_OR, a0, a1, a2);
2222	}
2223
2224	static void tgen_ori(TCGContext *s, TCGType type,
2225	TCGReg a0, TCGReg a1, tcg_target_long a2)
2226	{
2227	tcg_out_opc_imm(s, OPC_ORI, a0, a1, a2);
2228	}
2229
2230	static const TCGOutOpBinary outop_or = {
2231	.base.static_constraint = C_O1_I2(r, r, rI),
2232	.out_rrr = tgen_or,
2233	.out_rri = tgen_ori,
2234	};
2235
2236	static void tgen_orc(TCGContext *s, TCGType type,
2237	TCGReg a0, TCGReg a1, TCGReg a2)
2238	{
2239	tcg_out_opc_reg(s, OPC_ORN, a0, a1, a2);
2240	}
2241
2242	static const TCGOutOpBinary outop_orc = {
2243	.base.static_constraint = C_Dynamic,
2244	.base.dynamic_constraint = cset_zbb_rrr,
2245	.out_rrr = tgen_orc,
2246	};
2247
2248	static void tgen_rems(TCGContext *s, TCGType type,
2249	TCGReg a0, TCGReg a1, TCGReg a2)
2250	{
2251	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_REMW : OPC_REM;
2252	tcg_out_opc_reg(s, insn, a0, a1, a2);
2253	}
2254
2255	static const TCGOutOpBinary outop_rems = {
2256	.base.static_constraint = C_O1_I2(r, r, r),
2257	.out_rrr = tgen_rems,
2258	};
2259
2260	static void tgen_remu(TCGContext *s, TCGType type,
2261	TCGReg a0, TCGReg a1, TCGReg a2)
2262	{
2263	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_REMUW : OPC_REMU;
2264	tcg_out_opc_reg(s, insn, a0, a1, a2);
2265	}
2266
2267	static const TCGOutOpBinary outop_remu = {
2268	.base.static_constraint = C_O1_I2(r, r, r),
2269	.out_rrr = tgen_remu,
2270	};
2271
2272	static TCGConstraintSetIndex cset_rot(TCGType type, unsigned flags)
2273	{
2274	return cpuinfo & CPUINFO_ZBB ? C_O1_I2(r, r, ri) : C_NotImplemented;
2275	}
2276
2277	static void tgen_rotr(TCGContext *s, TCGType type,
2278	TCGReg a0, TCGReg a1, TCGReg a2)
2279	{
2280	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_RORW : OPC_ROR;
2281	tcg_out_opc_reg(s, insn, a0, a1, a2);
2282	}
2283
2284	static void tgen_rotri(TCGContext *s, TCGType type,
2285	TCGReg a0, TCGReg a1, tcg_target_long a2)
2286	{
2287	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_RORIW : OPC_RORI;
2288	unsigned mask = type == TCG_TYPE_I32 ? 31 : 63;
2289	tcg_out_opc_imm(s, insn, a0, a1, a2 & mask);
2290	}
2291
2292	static const TCGOutOpBinary outop_rotr = {
2293	.base.static_constraint = C_Dynamic,
2294	.base.dynamic_constraint = cset_rot,
2295	.out_rrr = tgen_rotr,
2296	.out_rri = tgen_rotri,
2297	};
2298
2299	static void tgen_rotl(TCGContext *s, TCGType type,
2300	TCGReg a0, TCGReg a1, TCGReg a2)
2301	{
2302	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_ROLW : OPC_ROL;
2303	tcg_out_opc_reg(s, insn, a0, a1, a2);
2304	}
2305
2306	static void tgen_rotli(TCGContext *s, TCGType type,
2307	TCGReg a0, TCGReg a1, tcg_target_long a2)
2308	{
2309	tgen_rotri(s, type, a0, a1, -a2);
2310	}
2311
2312	static const TCGOutOpBinary outop_rotl = {
2313	.base.static_constraint = C_Dynamic,
2314	.base.dynamic_constraint = cset_rot,
2315	.out_rrr = tgen_rotl,
2316	.out_rri = tgen_rotli,
2317	};
2318
2319	static void tgen_sar(TCGContext *s, TCGType type,
2320	TCGReg a0, TCGReg a1, TCGReg a2)
2321	{
2322	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SRAW : OPC_SRA;
2323	tcg_out_opc_reg(s, insn, a0, a1, a2);
2324	}
2325
2326	static void tgen_sari(TCGContext *s, TCGType type,
2327	TCGReg a0, TCGReg a1, tcg_target_long a2)
2328	{
2329	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SRAIW : OPC_SRAI;
2330	unsigned mask = type == TCG_TYPE_I32 ? 31 : 63;
2331	tcg_out_opc_imm(s, insn, a0, a1, a2 & mask);
2332	}
2333
2334	static const TCGOutOpBinary outop_sar = {
2335	.base.static_constraint = C_O1_I2(r, r, ri),
2336	.out_rrr = tgen_sar,
2337	.out_rri = tgen_sari,
2338	};
2339
2340	static void tgen_shl(TCGContext *s, TCGType type,
2341	TCGReg a0, TCGReg a1, TCGReg a2)
2342	{
2343	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SLLW : OPC_SLL;
2344	tcg_out_opc_reg(s, insn, a0, a1, a2);
2345	}
2346
2347	static void tgen_shli(TCGContext *s, TCGType type,
2348	TCGReg a0, TCGReg a1, tcg_target_long a2)
2349	{
2350	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SLLIW : OPC_SLLI;
2351	unsigned mask = type == TCG_TYPE_I32 ? 31 : 63;
2352	tcg_out_opc_imm(s, insn, a0, a1, a2 & mask);
2353	}
2354
2355	static const TCGOutOpBinary outop_shl = {
2356	.base.static_constraint = C_O1_I2(r, r, ri),
2357	.out_rrr = tgen_shl,
2358	.out_rri = tgen_shli,
2359	};
2360
2361	static void tgen_shr(TCGContext *s, TCGType type,
2362	TCGReg a0, TCGReg a1, TCGReg a2)
2363	{
2364	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SRLW : OPC_SRL;
2365	tcg_out_opc_reg(s, insn, a0, a1, a2);
2366	}
2367
2368	static void tgen_shri(TCGContext *s, TCGType type,
2369	TCGReg a0, TCGReg a1, tcg_target_long a2)
2370	{
2371	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SRLIW : OPC_SRLI;
2372	unsigned mask = type == TCG_TYPE_I32 ? 31 : 63;
2373	tcg_out_opc_imm(s, insn, a0, a1, a2 & mask);
2374	}
2375
2376	static const TCGOutOpBinary outop_shr = {
2377	.base.static_constraint = C_O1_I2(r, r, ri),
2378	.out_rrr = tgen_shr,
2379	.out_rri = tgen_shri,
2380	};
2381
2382	static void tgen_sub(TCGContext *s, TCGType type,
2383	TCGReg a0, TCGReg a1, TCGReg a2)
2384	{
2385	RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SUBW : OPC_SUB;
2386	tcg_out_opc_reg(s, insn, a0, a1, a2);
2387	}
2388
2389	static const TCGOutOpSubtract outop_sub = {
2390	.base.static_constraint = C_O1_I2(r, r, r),
2391	.out_rrr = tgen_sub,
2392	};
2393
2394	static const TCGOutOpAddSubCarry outop_subbo = {
2395	.base.static_constraint = C_NotImplemented,
2396	};
2397
2398	static const TCGOutOpAddSubCarry outop_subbi = {
2399	.base.static_constraint = C_NotImplemented,
2400	};
2401
2402	static const TCGOutOpAddSubCarry outop_subbio = {
2403	.base.static_constraint = C_NotImplemented,
2404	};
2405
2406	static void tcg_out_set_borrow(TCGContext *s)
2407	{
2408	g_assert_not_reached();
2409	}
2410
2411	static void tgen_smax(TCGContext *s, TCGType type,
2412	TCGReg a0, TCGReg a1, TCGReg a2)
2413	{
2414	tcg_out_opc_reg(s, OPC_MAX, a0, a1, a2);
2415	}
2416
2417	static const TCGOutOpBinary outop_smax = {
2418	.base.static_constraint = C_Dynamic,
2419	.base.dynamic_constraint = cset_zbb_rrr,
2420	.out_rrr = tgen_smax,
2421	};
2422
2423	static void tgen_smin(TCGContext *s, TCGType type,
2424	TCGReg a0, TCGReg a1, TCGReg a2)
2425	{
2426	tcg_out_opc_reg(s, OPC_MIN, a0, a1, a2);
2427	}
2428
2429	static const TCGOutOpBinary outop_smin = {
2430	.base.static_constraint = C_Dynamic,
2431	.base.dynamic_constraint = cset_zbb_rrr,
2432	.out_rrr = tgen_smin,
2433	};
2434
2435	static void tgen_umax(TCGContext *s, TCGType type,
2436	TCGReg a0, TCGReg a1, TCGReg a2)
2437	{
2438	tcg_out_opc_reg(s, OPC_MAXU, a0, a1, a2);
2439	}
2440
2441	static const TCGOutOpBinary outop_umax = {
2442	.base.static_constraint = C_Dynamic,
2443	.base.dynamic_constraint = cset_zbb_rrr,
2444	.out_rrr = tgen_umax,
2445	};
2446
2447	static void tgen_umin(TCGContext *s, TCGType type,
2448	TCGReg a0, TCGReg a1, TCGReg a2)
2449	{
2450	tcg_out_opc_reg(s, OPC_MINU, a0, a1, a2);
2451	}
2452
2453	static const TCGOutOpBinary outop_umin = {
2454	.base.static_constraint = C_Dynamic,
2455	.base.dynamic_constraint = cset_zbb_rrr,
2456	.out_rrr = tgen_umin,
2457	};
2458
2459	static void tgen_xor(TCGContext *s, TCGType type,
2460	TCGReg a0, TCGReg a1, TCGReg a2)
2461	{
2462	tcg_out_opc_reg(s, OPC_XOR, a0, a1, a2);
2463	}
2464
2465	static void tgen_xori(TCGContext *s, TCGType type,
2466	TCGReg a0, TCGReg a1, tcg_target_long a2)
2467	{
2468	tcg_out_opc_imm(s, OPC_XORI, a0, a1, a2);
2469	}
2470
2471	static const TCGOutOpBinary outop_xor = {
2472	.base.static_constraint = C_O1_I2(r, r, rI),
2473	.out_rrr = tgen_xor,
2474	.out_rri = tgen_xori,
2475	};
2476
2477	static TCGConstraintSetIndex cset_bswap(TCGType type, unsigned flags)
2478	{
2479	return cpuinfo & CPUINFO_ZBB ? C_O1_I1(r, r) : C_NotImplemented;
2480	}
2481
2482	static void tgen_bswap16(TCGContext *s, TCGType type,
2483	TCGReg a0, TCGReg a1, unsigned flags)
2484	{
2485	tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0);
2486	if (flags & TCG_BSWAP_OZ) {
2487	tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 48);
2488	} else {
2489	tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 48);
2490	}
2491	}
2492
2493	static const TCGOutOpBswap outop_bswap16 = {
2494	.base.static_constraint = C_Dynamic,
2495	.base.dynamic_constraint = cset_bswap,
2496	.out_rr = tgen_bswap16,
2497	};
2498
2499	static void tgen_bswap32(TCGContext *s, TCGType type,
2500	TCGReg a0, TCGReg a1, unsigned flags)
2501	{
2502	tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0);
2503	if (flags & TCG_BSWAP_OZ) {
2504	tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 32);
2505	} else {
2506	tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 32);
2507	}
2508	}
2509
2510	static const TCGOutOpBswap outop_bswap32 = {
2511	.base.static_constraint = C_Dynamic,
2512	.base.dynamic_constraint = cset_bswap,
2513	.out_rr = tgen_bswap32,
2514	};
2515
2516	static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2517	{
2518	tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0);
2519	}
2520
2521	static const TCGOutOpUnary outop_bswap64 = {
2522	.base.static_constraint = C_Dynamic,
2523	.base.dynamic_constraint = cset_bswap,
2524	.out_rr = tgen_bswap64,
2525	};
2526
2527	static TCGConstraintSetIndex cset_revbit8(TCGType type, unsigned flags)
2528	{
2529	return cpuinfo & CPUINFO_ZBKB ? C_O1_I1(r, r) : C_NotImplemented;
2530	}
2531
2532	static void tgen_revbit8(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2533	{
2534	tcg_out_opc_imm(s, OPC_BREV8, a0, a1, 0);
2535	}
2536
2537	static const TCGOutOpUnary outop_revbit8 = {
2538	.base.static_constraint = C_Dynamic,
2539	.base.dynamic_constraint = cset_revbit8,
2540	.out_rr = tgen_revbit8,
2541	};
2542
2543	static const TCGOutOpBswap outop_revbit32 = {
2544	.base.static_constraint = C_NotImplemented,
2545	};
2546
2547	static const TCGOutOpUnary outop_revbit64 = {
2548	.base.static_constraint = C_NotImplemented,
2549	};
2550
2551	static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2552	{
2553	tgen_sub(s, type, a0, TCG_REG_ZERO, a1);
2554	}
2555
2556	static const TCGOutOpUnary outop_neg = {
2557	.base.static_constraint = C_O1_I1(r, r),
2558	.out_rr = tgen_neg,
2559	};
2560
2561	static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2562	{
2563	tgen_xori(s, type, a0, a1, -1);
2564	}
2565
2566	static const TCGOutOpUnary outop_not = {
2567	.base.static_constraint = C_O1_I1(r, r),
2568	.out_rr = tgen_not,
2569	};
2570
2571	static const TCGOutOpDeposit outop_deposit = {
2572	.base.static_constraint = C_NotImplemented,
2573	};
2574
2575	static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2576	unsigned ofs, unsigned len)
2577	{
2578	if (ofs == 0) {
2579	switch (len) {
2580	case 16:
2581	tcg_out_ext16u(s, a0, a1);
2582	return;
2583	case 32:
2584	tcg_out_ext32u(s, a0, a1);
2585	return;
2586	}
2587	}
2588	if (ofs + len == 32) {
2589	tgen_shri(s, TCG_TYPE_I32, a0, a1, ofs);
2590	return;
2591	}
2592	if (len == 1) {
2593	tcg_out_opc_imm(s, OPC_BEXTI, a0, a1, ofs);
2594	return;
2595	}
2596	g_assert_not_reached();
2597	}
2598
2599	static const TCGOutOpExtract outop_extract = {
2600	.base.static_constraint = C_O1_I1(r, r),
2601	.out_rr = tgen_extract,
2602	};
2603
2604	static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2605	unsigned ofs, unsigned len)
2606	{
2607	if (ofs == 0) {
2608	switch (len) {
2609	case 8:
2610	tcg_out_ext8s(s, type, a0, a1);
2611	return;
2612	case 16:
2613	tcg_out_ext16s(s, type, a0, a1);
2614	return;
2615	case 32:
2616	tcg_out_ext32s(s, a0, a1);
2617	return;
2618	}
2619	} else if (ofs + len == 32) {
2620	tgen_sari(s, TCG_TYPE_I32, a0, a1, ofs);
2621	return;
2622	}
2623	g_assert_not_reached();
2624	}
2625
2626	static const TCGOutOpExtract outop_sextract = {
2627	.base.static_constraint = C_O1_I1(r, r),
2628	.out_rr = tgen_sextract,
2629	};
2630
2631	static const TCGOutOpExtract2 outop_extract2 = {
2632	.base.static_constraint = C_NotImplemented,
2633	};
2634
2635	static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest,
2636	TCGReg base, ptrdiff_t offset)
2637	{
2638	tcg_out_ldst(s, OPC_LBU, dest, base, offset);
2639	}
2640
2641	static const TCGOutOpLoad outop_ld8u = {
2642	.base.static_constraint = C_O1_I1(r, r),
2643	.out = tgen_ld8u,
2644	};
2645
2646	static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest,
2647	TCGReg base, ptrdiff_t offset)
2648	{
2649	tcg_out_ldst(s, OPC_LB, dest, base, offset);
2650	}
2651
2652	static const TCGOutOpLoad outop_ld8s = {
2653	.base.static_constraint = C_O1_I1(r, r),
2654	.out = tgen_ld8s,
2655	};
2656
2657	static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest,
2658	TCGReg base, ptrdiff_t offset)
2659	{
2660	tcg_out_ldst(s, OPC_LHU, dest, base, offset);
2661	}
2662
2663	static const TCGOutOpLoad outop_ld16u = {
2664	.base.static_constraint = C_O1_I1(r, r),
2665	.out = tgen_ld16u,
2666	};
2667
2668	static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest,
2669	TCGReg base, ptrdiff_t offset)
2670	{
2671	tcg_out_ldst(s, OPC_LH, dest, base, offset);
2672	}
2673
2674	static const TCGOutOpLoad outop_ld16s = {
2675	.base.static_constraint = C_O1_I1(r, r),
2676	.out = tgen_ld16s,
2677	};
2678
2679	static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest,
2680	TCGReg base, ptrdiff_t offset)
2681	{
2682	tcg_out_ldst(s, OPC_LWU, dest, base, offset);
2683	}
2684
2685	static const TCGOutOpLoad outop_ld32u = {
2686	.base.static_constraint = C_O1_I1(r, r),
2687	.out = tgen_ld32u,
2688	};
2689
2690	static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest,
2691	TCGReg base, ptrdiff_t offset)
2692	{
2693	tcg_out_ldst(s, OPC_LW, dest, base, offset);
2694	}
2695
2696	static const TCGOutOpLoad outop_ld32s = {
2697	.base.static_constraint = C_O1_I1(r, r),
2698	.out = tgen_ld32s,
2699	};
2700
2701	static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data,
2702	TCGReg base, ptrdiff_t offset)
2703	{
2704	tcg_out_ldst(s, OPC_SB, data, base, offset);
2705	}
2706
2707	static const TCGOutOpStore outop_st8 = {
2708	.base.static_constraint = C_O0_I2(rz, r),
2709	.out_r = tgen_st8_r,
2710	};
2711
2712	static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data,
2713	TCGReg base, ptrdiff_t offset)
2714	{
2715	tcg_out_ldst(s, OPC_SH, data, base, offset);
2716	}
2717
2718	static const TCGOutOpStore outop_st16 = {
2719	.base.static_constraint = C_O0_I2(rz, r),
2720	.out_r = tgen_st16_r,
2721	};
2722
2723	static const TCGOutOpStore outop_st = {
2724	.base.static_constraint = C_O0_I2(rz, r),
2725	.out_r = tcg_out_st,
2726	};
2727
2728
2729	static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2730	unsigned vecl, unsigned vece,
2731	const TCGArg args[TCG_MAX_OP_ARGS],
2732	const int const_args[TCG_MAX_OP_ARGS])
2733	{
2734	TCGType type = vecl + TCG_TYPE_V64;
2735	TCGArg a0, a1, a2;
2736	int c2;
2737
2738	a0 = args[0];
2739	a1 = args[1];
2740	a2 = args[2];
2741	c2 = const_args[2];
2742
2743	switch (opc) {
2744	case INDEX_op_dupm_vec:
2745	tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
2746	break;
2747	case INDEX_op_ld_vec:
2748	tcg_out_ld(s, type, a0, a1, a2);
2749	break;
2750	case INDEX_op_st_vec:
2751	tcg_out_st(s, type, a0, a1, a2);
2752	break;
2753	case INDEX_op_add_vec:
2754	set_vtype_len_sew(s, type, vece);
2755	tcg_out_opc_vv_vi(s, OPC_VADD_VV, OPC_VADD_VI, a0, a1, a2, c2);
2756	break;
2757	case INDEX_op_sub_vec:
2758	set_vtype_len_sew(s, type, vece);
2759	if (const_args[1]) {
2760	tcg_out_opc_vi(s, OPC_VRSUB_VI, a0, a2, a1);
2761	} else {
2762	tcg_out_opc_vv(s, OPC_VSUB_VV, a0, a1, a2);
2763	}
2764	break;
2765	case INDEX_op_and_vec:
2766	set_vtype_len(s, type);
2767	tcg_out_opc_vv_vi(s, OPC_VAND_VV, OPC_VAND_VI, a0, a1, a2, c2);
2768	break;
2769	case INDEX_op_or_vec:
2770	set_vtype_len(s, type);
2771	tcg_out_opc_vv_vi(s, OPC_VOR_VV, OPC_VOR_VI, a0, a1, a2, c2);
2772	break;
2773	case INDEX_op_xor_vec:
2774	set_vtype_len(s, type);
2775	tcg_out_opc_vv_vi(s, OPC_VXOR_VV, OPC_VXOR_VI, a0, a1, a2, c2);
2776	break;
2777	case INDEX_op_not_vec:
2778	set_vtype_len(s, type);
2779	tcg_out_opc_vi(s, OPC_VXOR_VI, a0, a1, -1);
2780	break;
2781	case INDEX_op_neg_vec:
2782	set_vtype_len_sew(s, type, vece);
2783	tcg_out_opc_vi(s, OPC_VRSUB_VI, a0, a1, 0);
2784	break;
2785	case INDEX_op_mul_vec:
2786	set_vtype_len_sew(s, type, vece);
2787	tcg_out_opc_vv(s, OPC_VMUL_VV, a0, a1, a2);
2788	break;
2789	case INDEX_op_ssadd_vec:
2790	set_vtype_len_sew(s, type, vece);
2791	tcg_out_opc_vv_vi(s, OPC_VSADD_VV, OPC_VSADD_VI, a0, a1, a2, c2);
2792	break;
2793	case INDEX_op_sssub_vec:
2794	set_vtype_len_sew(s, type, vece);
2795	tcg_out_opc_vv_vi(s, OPC_VSSUB_VV, OPC_VSSUB_VI, a0, a1, a2, c2);
2796	break;
2797	case INDEX_op_usadd_vec:
2798	set_vtype_len_sew(s, type, vece);
2799	tcg_out_opc_vv_vi(s, OPC_VSADDU_VV, OPC_VSADDU_VI, a0, a1, a2, c2);
2800	break;
2801	case INDEX_op_ussub_vec:
2802	set_vtype_len_sew(s, type, vece);
2803	tcg_out_opc_vv_vi(s, OPC_VSSUBU_VV, OPC_VSSUBU_VI, a0, a1, a2, c2);
2804	break;
2805	case INDEX_op_smax_vec:
2806	set_vtype_len_sew(s, type, vece);
2807	tcg_out_opc_vv_vi(s, OPC_VMAX_VV, OPC_VMAX_VI, a0, a1, a2, c2);
2808	break;
2809	case INDEX_op_smin_vec:
2810	set_vtype_len_sew(s, type, vece);
2811	tcg_out_opc_vv_vi(s, OPC_VMIN_VV, OPC_VMIN_VI, a0, a1, a2, c2);
2812	break;
2813	case INDEX_op_umax_vec:
2814	set_vtype_len_sew(s, type, vece);
2815	tcg_out_opc_vv_vi(s, OPC_VMAXU_VV, OPC_VMAXU_VI, a0, a1, a2, c2);
2816	break;
2817	case INDEX_op_umin_vec:
2818	set_vtype_len_sew(s, type, vece);
2819	tcg_out_opc_vv_vi(s, OPC_VMINU_VV, OPC_VMINU_VI, a0, a1, a2, c2);
2820	break;
2821	case INDEX_op_shls_vec:
2822	set_vtype_len_sew(s, type, vece);
2823	tcg_out_opc_vx(s, OPC_VSLL_VX, a0, a1, a2);
2824	break;
2825	case INDEX_op_shrs_vec:
2826	set_vtype_len_sew(s, type, vece);
2827	tcg_out_opc_vx(s, OPC_VSRL_VX, a0, a1, a2);
2828	break;
2829	case INDEX_op_sars_vec:
2830	set_vtype_len_sew(s, type, vece);
2831	tcg_out_opc_vx(s, OPC_VSRA_VX, a0, a1, a2);
2832	break;
2833	case INDEX_op_shlv_vec:
2834	set_vtype_len_sew(s, type, vece);
2835	tcg_out_opc_vv(s, OPC_VSLL_VV, a0, a1, a2);
2836	break;
2837	case INDEX_op_shrv_vec:
2838	set_vtype_len_sew(s, type, vece);
2839	tcg_out_opc_vv(s, OPC_VSRL_VV, a0, a1, a2);
2840	break;
2841	case INDEX_op_sarv_vec:
2842	set_vtype_len_sew(s, type, vece);
2843	tcg_out_opc_vv(s, OPC_VSRA_VV, a0, a1, a2);
2844	break;
2845	case INDEX_op_shli_vec:
2846	set_vtype_len_sew(s, type, vece);
2847	tcg_out_vshifti(s, OPC_VSLL_VI, OPC_VSLL_VX, a0, a1, a2);
2848	break;
2849	case INDEX_op_shri_vec:
2850	set_vtype_len_sew(s, type, vece);
2851	tcg_out_vshifti(s, OPC_VSRL_VI, OPC_VSRL_VX, a0, a1, a2);
2852	break;
2853	case INDEX_op_sari_vec:
2854	set_vtype_len_sew(s, type, vece);
2855	tcg_out_vshifti(s, OPC_VSRA_VI, OPC_VSRA_VX, a0, a1, a2);
2856	break;
2857	case INDEX_op_rotli_vec:
2858	set_vtype_len_sew(s, type, vece);
2859	tcg_out_vshifti(s, OPC_VSLL_VI, OPC_VSLL_VX, TCG_REG_V0, a1, a2);
2860	tcg_out_vshifti(s, OPC_VSRL_VI, OPC_VSRL_VX, a0, a1,
2861	-a2 & ((8 << vece) - 1));
2862	tcg_out_opc_vv(s, OPC_VOR_VV, a0, a0, TCG_REG_V0);
2863	break;
2864	case INDEX_op_rotls_vec:
2865	set_vtype_len_sew(s, type, vece);
2866	tcg_out_opc_vx(s, OPC_VSLL_VX, TCG_REG_V0, a1, a2);
2867	tcg_out_opc_reg(s, OPC_SUBW, TCG_REG_TMP0, TCG_REG_ZERO, a2);
2868	tcg_out_opc_vx(s, OPC_VSRL_VX, a0, a1, TCG_REG_TMP0);
2869	tcg_out_opc_vv(s, OPC_VOR_VV, a0, a0, TCG_REG_V0);
2870	break;
2871	case INDEX_op_rotlv_vec:
2872	set_vtype_len_sew(s, type, vece);
2873	tcg_out_opc_vi(s, OPC_VRSUB_VI, TCG_REG_V0, a2, 0);
2874	tcg_out_opc_vv(s, OPC_VSRL_VV, TCG_REG_V0, a1, TCG_REG_V0);
2875	tcg_out_opc_vv(s, OPC_VSLL_VV, a0, a1, a2);
2876	tcg_out_opc_vv(s, OPC_VOR_VV, a0, a0, TCG_REG_V0);
2877	break;
2878	case INDEX_op_rotrv_vec:
2879	set_vtype_len_sew(s, type, vece);
2880	tcg_out_opc_vi(s, OPC_VRSUB_VI, TCG_REG_V0, a2, 0);
2881	tcg_out_opc_vv(s, OPC_VSLL_VV, TCG_REG_V0, a1, TCG_REG_V0);
2882	tcg_out_opc_vv(s, OPC_VSRL_VV, a0, a1, a2);
2883	tcg_out_opc_vv(s, OPC_VOR_VV, a0, a0, TCG_REG_V0);
2884	break;
2885	case INDEX_op_cmp_vec:
2886	tcg_out_cmpsel(s, type, vece, args[3], a0, a1, a2, c2,
2887	-1, true, 0, true);
2888	break;
2889	case INDEX_op_cmpsel_vec:
2890	tcg_out_cmpsel(s, type, vece, args[5], a0, a1, a2, c2,
2891	args[3], const_args[3], args[4], const_args[4]);
2892	break;
2893	case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
2894	case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
2895	default:
2896	g_assert_not_reached();
2897	}
2898	}
2899
2900	void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
2901	TCGArg a0, ...)
2902	{
2903	g_assert_not_reached();
2904	}
2905
2906	int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
2907	{
2908	switch (opc) {
2909	case INDEX_op_add_vec:
2910	case INDEX_op_sub_vec:
2911	case INDEX_op_and_vec:
2912	case INDEX_op_or_vec:
2913	case INDEX_op_xor_vec:
2914	case INDEX_op_not_vec:
2915	case INDEX_op_neg_vec:
2916	case INDEX_op_mul_vec:
2917	case INDEX_op_ssadd_vec:
2918	case INDEX_op_sssub_vec:
2919	case INDEX_op_usadd_vec:
2920	case INDEX_op_ussub_vec:
2921	case INDEX_op_smax_vec:
2922	case INDEX_op_smin_vec:
2923	case INDEX_op_umax_vec:
2924	case INDEX_op_umin_vec:
2925	case INDEX_op_shls_vec:
2926	case INDEX_op_shrs_vec:
2927	case INDEX_op_sars_vec:
2928	case INDEX_op_shlv_vec:
2929	case INDEX_op_shrv_vec:
2930	case INDEX_op_sarv_vec:
2931	case INDEX_op_shri_vec:
2932	case INDEX_op_shli_vec:
2933	case INDEX_op_sari_vec:
2934	case INDEX_op_rotls_vec:
2935	case INDEX_op_rotlv_vec:
2936	case INDEX_op_rotrv_vec:
2937	case INDEX_op_rotli_vec:
2938	case INDEX_op_cmp_vec:
2939	case INDEX_op_cmpsel_vec:
2940	return 1;
2941	default:
2942	return 0;
2943	}
2944	}
2945
2946	static TCGConstraintSetIndex
2947	tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags)
2948	{
2949	switch (op) {
2950	case INDEX_op_st_vec:
2951	return C_O0_I2(v, r);
2952	case INDEX_op_dup_vec:
2953	case INDEX_op_dupm_vec:
2954	case INDEX_op_ld_vec:
2955	return C_O1_I1(v, r);
2956	case INDEX_op_neg_vec:
2957	case INDEX_op_not_vec:
2958	case INDEX_op_shli_vec:
2959	case INDEX_op_shri_vec:
2960	case INDEX_op_sari_vec:
2961	case INDEX_op_rotli_vec:
2962	return C_O1_I1(v, v);
2963	case INDEX_op_add_vec:
2964	case INDEX_op_and_vec:
2965	case INDEX_op_or_vec:
2966	case INDEX_op_xor_vec:
2967	case INDEX_op_ssadd_vec:
2968	case INDEX_op_sssub_vec:
2969	case INDEX_op_usadd_vec:
2970	case INDEX_op_ussub_vec:
2971	case INDEX_op_smax_vec:
2972	case INDEX_op_smin_vec:
2973	case INDEX_op_umax_vec:
2974	case INDEX_op_umin_vec:
2975	return C_O1_I2(v, v, vK);
2976	case INDEX_op_sub_vec:
2977	return C_O1_I2(v, vK, v);
2978	case INDEX_op_mul_vec:
2979	case INDEX_op_shlv_vec:
2980	case INDEX_op_shrv_vec:
2981	case INDEX_op_sarv_vec:
2982	case INDEX_op_rotlv_vec:
2983	case INDEX_op_rotrv_vec:
2984	return C_O1_I2(v, v, v);
2985	case INDEX_op_shls_vec:
2986	case INDEX_op_shrs_vec:
2987	case INDEX_op_sars_vec:
2988	case INDEX_op_rotls_vec:
2989	return C_O1_I2(v, v, r);
2990	case INDEX_op_cmp_vec:
2991	return C_O1_I2(v, v, vL);
2992	case INDEX_op_cmpsel_vec:
2993	return C_O1_I4(v, v, vL, vK, vK);
2994	default:
2995	return C_NotImplemented;
2996	}
2997	}
2998
2999	static const int tcg_target_callee_save_regs[] = {
3000	TCG_REG_S0, /* used for the global env (TCG_AREG0) */
3001	TCG_REG_S1,
3002	TCG_REG_S2,
3003	TCG_REG_S3,
3004	TCG_REG_S4,
3005	TCG_REG_S5,
3006	TCG_REG_S6,
3007	TCG_REG_S7,
3008	TCG_REG_S8,
3009	TCG_REG_S9,
3010	TCG_REG_S10,
3011	TCG_REG_S11,
3012	TCG_REG_RA, /* should be last for ABI compliance */
3013	};
3014
3015	/* Stack frame parameters. */
3016	#define REG_SIZE ((int)sizeof(tcg_target_long))
3017	#define SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * REG_SIZE)
3018	#define TEMP_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long))
3019	#define FRAME_SIZE ((TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE + SAVE_SIZE \
3020	+ TCG_TARGET_STACK_ALIGN - 1) \
3021	& -TCG_TARGET_STACK_ALIGN)
3022	#define SAVE_OFS (TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE)
3023
3024	/* We're expecting to be able to use an immediate for frame allocation. */
3025	QEMU_BUILD_BUG_ON(FRAME_SIZE > 0x7ff);
3026
3027	/* Generate global QEMU prologue and epilogue code */
3028	static void tcg_target_qemu_prologue(TCGContext *s)
3029	{
3030	int i;
3031
3032	tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE, TEMP_SIZE);
3033
3034	/* TB prologue */
3035	tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_SP, TCG_REG_SP, -FRAME_SIZE);
3036	for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
3037	tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
3038	TCG_REG_SP, SAVE_OFS + i * REG_SIZE);
3039	}
3040
3041	if (!tcg_use_softmmu && guest_base) {
3042	tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
3043	tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
3044	}
3045
3046	/* Call generated code */
3047	tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
3048	tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0);
3049
3050	/* Return path for goto_ptr. Set return value to 0 */
3051	tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
3052	tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO);
3053
3054	/* TB epilogue */
3055	tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
3056	for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
3057	tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
3058	TCG_REG_SP, SAVE_OFS + i * REG_SIZE);
3059	}
3060
3061	tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_SP, TCG_REG_SP, FRAME_SIZE);
3062	tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_RA, 0);
3063	}
3064
3065	static void tcg_out_tb_start(TCGContext *s)
3066	{
3067	init_setting_vtype(s);
3068	}
3069
3070	static bool vtype_check(unsigned vtype)
3071	{
3072	unsigned long tmp;
3073
3074	/* vsetvl tmp, zero, vtype */
3075	asm(".insn r 0x57, 7, 0x40, %0, zero, %1" : "=r"(tmp) : "r"(vtype));
3076	return tmp != 0;
3077	}
3078
3079	static void probe_frac_lmul_1(TCGType type, MemOp vsew)
3080	{
3081	VsetCache *p = &riscv_vset_cache[type - TCG_TYPE_V64][vsew];
3082	unsigned avl = tcg_type_size(type) >> vsew;
3083	int lmul = type - riscv_lg2_vlenb;
3084	unsigned vtype = encode_vtype(true, true, vsew, lmul & 7);
3085	bool lmul_eq_avl = true;
3086
3087	/* Guaranteed by Zve64x. */
3088	assert(lmul < 3);
3089
3090	/*
3091	* For LMUL < -3, the host vector size is so large that TYPE
3092	* is smaller than the minimum 1/8 fraction.
3093	*
3094	* For other fractional LMUL settings, implementations must
3095	* support SEW settings between SEW_MIN and LMUL * ELEN, inclusive.
3096	* So if ELEN = 64, LMUL = 1/2, then SEW will support e8, e16, e32,
3097	* but e64 may not be supported. In other words, the hardware only
3098	* guarantees SEW_MIN <= SEW <= LMUL * ELEN. Check.
3099	*/
3100	if (lmul < 0 && (lmul < -3 \|\| !vtype_check(vtype))) {
3101	vtype = encode_vtype(true, true, vsew, VLMUL_M1);
3102	lmul_eq_avl = false;
3103	}
3104
3105	if (avl < 32) {
3106	p->vset_insn = encode_vseti(OPC_VSETIVLI, TCG_REG_ZERO, avl, vtype);
3107	} else if (lmul_eq_avl) {
3108	/* rd != 0 and rs1 == 0 uses vlmax */
3109	p->vset_insn = encode_vset(OPC_VSETVLI, TCG_REG_TMP0, TCG_REG_ZERO, vtype);
3110	} else {
3111	p->movi_insn = encode_i(OPC_ADDI, TCG_REG_TMP0, TCG_REG_ZERO, avl);
3112	p->vset_insn = encode_vset(OPC_VSETVLI, TCG_REG_ZERO, TCG_REG_TMP0, vtype);
3113	}
3114	}
3115
3116	static void probe_frac_lmul(void)
3117	{
3118	/* Match riscv_lg2_vlenb to TCG_TYPE_V64. */
3119	QEMU_BUILD_BUG_ON(TCG_TYPE_V64 != 3);
3120
3121	for (TCGType t = TCG_TYPE_V64; t <= TCG_TYPE_V256; t++) {
3122	for (MemOp e = MO_8; e <= MO_64; e++) {
3123	probe_frac_lmul_1(t, e);
3124	}
3125	}
3126	}
3127
3128	static void tcg_target_init(TCGContext *s)
3129	{
3130	tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff;
3131	tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff;
3132
3133	tcg_target_call_clobber_regs = -1;
3134	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0);
3135	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S1);
3136	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S2);
3137	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S3);
3138	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S4);
3139	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S5);
3140	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S6);
3141	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S7);
3142	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S8);
3143	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S9);
3144	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S10);
3145	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S11);
3146
3147	s->reserved_regs = 0;
3148	tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO);
3149	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP0);
3150	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1);
3151	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP2);
3152	tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
3153	tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP);
3154	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TP);
3155
3156	if (cpuinfo & CPUINFO_ZVE64X) {
3157	switch (riscv_lg2_vlenb) {
3158	case TCG_TYPE_V64:
3159	tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS;
3160	tcg_target_available_regs[TCG_TYPE_V128] = ALL_DVECTOR_REG_GROUPS;
3161	tcg_target_available_regs[TCG_TYPE_V256] = ALL_QVECTOR_REG_GROUPS;
3162	s->reserved_regs \|= (~ALL_QVECTOR_REG_GROUPS & ALL_VECTOR_REGS);
3163	break;
3164	case TCG_TYPE_V128:
3165	tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS;
3166	tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS;
3167	tcg_target_available_regs[TCG_TYPE_V256] = ALL_DVECTOR_REG_GROUPS;
3168	s->reserved_regs \|= (~ALL_DVECTOR_REG_GROUPS & ALL_VECTOR_REGS);
3169	break;
3170	default:
3171	/* Guaranteed by Zve64x. */
3172	tcg_debug_assert(riscv_lg2_vlenb >= TCG_TYPE_V256);
3173	tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS;
3174	tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS;
3175	tcg_target_available_regs[TCG_TYPE_V256] = ALL_VECTOR_REGS;
3176	break;
3177	}
3178	tcg_regset_set_reg(s->reserved_regs, TCG_REG_V0);
3179	probe_frac_lmul();
3180	}
3181	}
3182
3183	typedef struct {
3184	DebugFrameHeader h;
3185	uint8_t fde_def_cfa[4];
3186	uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2];
3187	} DebugFrame;
3188
3189	#define ELF_HOST_MACHINE EM_RISCV
3190
3191	static const DebugFrame debug_frame = {
3192	.h.cie.len = sizeof(DebugFrameCIE) - 4, /* length after .len member */
3193	.h.cie.id = -1,
3194	.h.cie.version = 1,
3195	.h.cie.code_align = 1,
3196	.h.cie.data_align = -sizeof(tcg_target_long) & 0x7f, /* sleb128 */
3197	.h.cie.return_column = TCG_REG_RA,
3198
3199	/* Total FDE size does not include the "len" member. */
3200	.h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
3201
3202	.fde_def_cfa = {
3203	12, TCG_REG_SP, /* DW_CFA_def_cfa sp, ... */
3204	(FRAME_SIZE & 0x7f) \| 0x80, /* ... uleb128 FRAME_SIZE */
3205	(FRAME_SIZE >> 7)
3206	},
3207	.fde_reg_ofs = {
3208	0x80 + 9, 12, /* DW_CFA_offset, s1, -96 */
3209	0x80 + 18, 11, /* DW_CFA_offset, s2, -88 */
3210	0x80 + 19, 10, /* DW_CFA_offset, s3, -80 */
3211	0x80 + 20, 9, /* DW_CFA_offset, s4, -72 */
3212	0x80 + 21, 8, /* DW_CFA_offset, s5, -64 */
3213	0x80 + 22, 7, /* DW_CFA_offset, s6, -56 */
3214	0x80 + 23, 6, /* DW_CFA_offset, s7, -48 */
3215	0x80 + 24, 5, /* DW_CFA_offset, s8, -40 */
3216	0x80 + 25, 4, /* DW_CFA_offset, s9, -32 */
3217	0x80 + 26, 3, /* DW_CFA_offset, s10, -24 */
3218	0x80 + 27, 2, /* DW_CFA_offset, s11, -16 */
3219	0x80 + 1 , 1, /* DW_CFA_offset, ra, -8 */
3220	}
3221	};
3222
3223	void tcg_register_jit(const void *buf, size_t buf_size)
3224	{
3225	tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
3226	}