tcg-target.c.inc · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

inc 3,760 lines 116 KB

Raw

1	/*
2	* Initial TCG Implementation for aarch64
3	*
4	* Copyright (c) 2013 Huawei Technologies Duesseldorf GmbH
5	* Written by Claudio Fontana
6	*
7	* This work is licensed under the terms of the GNU GPL, version 2 or
8	* (at your option) any later version.
9	*
10	* See the COPYING file in the top-level directory for details.
11	*/
12
13	#include "qemu/bitops.h"
14
15	/* Used for function call generation. */
16	#define TCG_REG_CALL_STACK TCG_REG_SP
17	#define TCG_TARGET_STACK_ALIGN 16
18	#define TCG_TARGET_CALL_STACK_OFFSET 0
19	#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL
20	#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL
21	#ifdef CONFIG_DARWIN
22	# define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL
23	#else
24	# define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_EVEN
25	#endif
26	#define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL
27
28	/* We're going to re-use TCGType in setting of the SF bit, which controls
29	the size of the operation performed. If we know the values match, it
30	makes things much cleaner. */
31	QEMU_BUILD_BUG_ON(TCG_TYPE_I32 != 0 \|\| TCG_TYPE_I64 != 1);
32
33	#ifdef CONFIG_DEBUG_TCG
34	static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
35	"x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
36	"x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
37	"x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
38	"x24", "x25", "x26", "x27", "x28", "fp", "x30", "sp",
39
40	"v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
41	"v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
42	"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
43	"v24", "v25", "v26", "v27", "v28", "fp", "v30", "v31",
44	};
45	#endif /* CONFIG_DEBUG_TCG */
46
47	static const int tcg_target_reg_alloc_order[] = {
48	TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
49	TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
50	TCG_REG_X28, /* we will reserve this for guest_base if configured */
51
52	TCG_REG_X8, TCG_REG_X9, TCG_REG_X10, TCG_REG_X11,
53	TCG_REG_X12, TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
54
55	TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
56	TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7,
57
58	/* X16 reserved as temporary */
59	/* X17 reserved as temporary */
60	/* X18 reserved by system */
61	/* X19 reserved for AREG0 */
62	/* X29 reserved as fp */
63	/* X30 reserved as temporary */
64
65	TCG_REG_V0, TCG_REG_V1, TCG_REG_V2, TCG_REG_V3,
66	TCG_REG_V4, TCG_REG_V5, TCG_REG_V6, TCG_REG_V7,
67	/* V8 - V15 are call-saved, and skipped. */
68	TCG_REG_V16, TCG_REG_V17, TCG_REG_V18, TCG_REG_V19,
69	TCG_REG_V20, TCG_REG_V21, TCG_REG_V22, TCG_REG_V23,
70	TCG_REG_V24, TCG_REG_V25, TCG_REG_V26, TCG_REG_V27,
71	TCG_REG_V28, TCG_REG_V29, TCG_REG_V30, TCG_REG_V31,
72	};
73
74	static const int tcg_target_call_iarg_regs[8] = {
75	TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
76	TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7
77	};
78
79	static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot)
80	{
81	tcg_debug_assert(kind == TCG_CALL_RET_NORMAL);
82	tcg_debug_assert(slot >= 0 && slot <= 1);
83	return TCG_REG_X0 + slot;
84	}
85
86	#define TCG_REG_TMP0 TCG_REG_X16
87	#define TCG_REG_TMP1 TCG_REG_X17
88	#define TCG_REG_TMP2 TCG_REG_X30
89	#define TCG_VEC_TMP0 TCG_REG_V31
90
91	#define TCG_REG_GUEST_BASE TCG_REG_X28
92
93	static bool reloc_pc26(tcg_insn_unit src_rw, const tcg_insn_unit target)
94	{
95	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
96	ptrdiff_t offset = target - src_rx;
97
98	if (offset == sextract64(offset, 0, 26)) {
99	/* read instruction, mask away previous PC_REL26 parameter contents,
100	set the proper offset, then write back the instruction. */
101	src_rw = deposit32(src_rw, 0, 26, offset);
102	return true;
103	}
104	return false;
105	}
106
107	static bool reloc_pc19(tcg_insn_unit src_rw, const tcg_insn_unit target)
108	{
109	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
110	ptrdiff_t offset = target - src_rx;
111
112	if (offset == sextract64(offset, 0, 19)) {
113	src_rw = deposit32(src_rw, 5, 19, offset);
114	return true;
115	}
116	return false;
117	}
118
119	static bool reloc_pc14(tcg_insn_unit src_rw, const tcg_insn_unit target)
120	{
121	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
122	ptrdiff_t offset = target - src_rx;
123
124	if (offset == sextract64(offset, 0, 14)) {
125	src_rw = deposit32(src_rw, 5, 14, offset);
126	return true;
127	}
128	return false;
129	}
130
131	static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
132	intptr_t value, intptr_t addend)
133	{
134	tcg_debug_assert(addend == 0);
135	switch (type) {
136	case R_AARCH64_JUMP26:
137	case R_AARCH64_CALL26:
138	return reloc_pc26(code_ptr, (const tcg_insn_unit *)value);
139	case R_AARCH64_CONDBR19:
140	return reloc_pc19(code_ptr, (const tcg_insn_unit *)value);
141	case R_AARCH64_TSTBR14:
142	return reloc_pc14(code_ptr, (const tcg_insn_unit *)value);
143	default:
144	g_assert_not_reached();
145	}
146	}
147
148	#define TCG_CT_CONST_AIMM 0x100
149	#define TCG_CT_CONST_LIMM 0x200
150	#define TCG_CT_CONST_ZERO 0x400
151	#define TCG_CT_CONST_MONE 0x800
152	#define TCG_CT_CONST_ORRI 0x1000
153	#define TCG_CT_CONST_ANDI 0x2000
154	#define TCG_CT_CONST_CMP 0x4000
155	#define TCG_CT_CONST_S8 0x8000
156	#define TCG_CT_CONST_U8 0x10000
157
158	#define ALL_GENERAL_REGS 0xffffffffu
159	#define ALL_VECTOR_REGS 0xffffffff00000000ull
160
161	/* Match a constant valid for addition (12-bit, optionally shifted). */
162	static inline bool is_aimm(uint64_t val)
163	{
164	return (val & ~0xfff) == 0 \|\| (val & ~0xfff000) == 0;
165	}
166
167	/* Match a constant valid for logical operations. */
168	static inline bool is_limm(uint64_t val)
169	{
170	/* Taking a simplified view of the logical immediates for now, ignoring
171	the replication that can happen across the field. Match bit patterns
172	of the forms
173	0....01....1
174	0..01..10..0
175	and their inverses. */
176
177	/* Make things easier below, by testing the form with msb clear. */
178	if ((int64_t)val < 0) {
179	val = ~val;
180	}
181	if (val == 0) {
182	return false;
183	}
184	val += val & -val;
185	return (val & (val - 1)) == 0;
186	}
187
188	/* Return true if v16 is a valid 16-bit shifted immediate. */
189	static bool is_shimm16(uint16_t v16, int cmode, int imm8)
190	{
191	if (v16 == (v16 & 0xff)) {
192	*cmode = 0x8;
193	*imm8 = v16 & 0xff;
194	return true;
195	} else if (v16 == (v16 & 0xff00)) {
196	*cmode = 0xa;
197	*imm8 = v16 >> 8;
198	return true;
199	}
200	return false;
201	}
202
203	/* Return true if v32 is a valid 32-bit shifted immediate. */
204	static bool is_shimm32(uint32_t v32, int cmode, int imm8)
205	{
206	if (v32 == (v32 & 0xff)) {
207	*cmode = 0x0;
208	*imm8 = v32 & 0xff;
209	return true;
210	} else if (v32 == (v32 & 0xff00)) {
211	*cmode = 0x2;
212	*imm8 = (v32 >> 8) & 0xff;
213	return true;
214	} else if (v32 == (v32 & 0xff0000)) {
215	*cmode = 0x4;
216	*imm8 = (v32 >> 16) & 0xff;
217	return true;
218	} else if (v32 == (v32 & 0xff000000)) {
219	*cmode = 0x6;
220	*imm8 = v32 >> 24;
221	return true;
222	}
223	return false;
224	}
225
226	/* Return true if v32 is a valid 32-bit shifting ones immediate. */
227	static bool is_soimm32(uint32_t v32, int cmode, int imm8)
228	{
229	if ((v32 & 0xffff00ff) == 0xff) {
230	*cmode = 0xc;
231	*imm8 = (v32 >> 8) & 0xff;
232	return true;
233	} else if ((v32 & 0xff00ffff) == 0xffff) {
234	*cmode = 0xd;
235	*imm8 = (v32 >> 16) & 0xff;
236	return true;
237	}
238	return false;
239	}
240
241	/* Return true if v32 is a valid float32 immediate. */
242	static bool is_fimm32(uint32_t v32, int cmode, int imm8)
243	{
244	if (extract32(v32, 0, 19) == 0
245	&& (extract32(v32, 25, 6) == 0x20
246	\|\| extract32(v32, 25, 6) == 0x1f)) {
247	*cmode = 0xf;
248	*imm8 = (extract32(v32, 31, 1) << 7)
249	\| (extract32(v32, 25, 1) << 6)
250	\| extract32(v32, 19, 6);
251	return true;
252	}
253	return false;
254	}
255
256	/* Return true if v64 is a valid float64 immediate. */
257	static bool is_fimm64(uint64_t v64, int cmode, int imm8)
258	{
259	if (extract64(v64, 0, 48) == 0
260	&& (extract64(v64, 54, 9) == 0x100
261	\|\| extract64(v64, 54, 9) == 0x0ff)) {
262	*cmode = 0xf;
263	*imm8 = (extract64(v64, 63, 1) << 7)
264	\| (extract64(v64, 54, 1) << 6)
265	\| extract64(v64, 48, 6);
266	return true;
267	}
268	return false;
269	}
270
271	/*
272	* Return non-zero if v32 can be formed by MOVI+ORR.
273	* Place the parameters for MOVI in (cmode, imm8).
274	* Return the cmode for ORR; the imm8 can be had via extraction from v32.
275	*/
276	static int is_shimm32_pair(uint32_t v32, int cmode, int imm8)
277	{
278	int i;
279
280	for (i = 6; i > 0; i -= 2) {
281	/* Mask out one byte we can add with ORR. */
282	uint32_t tmp = v32 & ~(0xffu << (i * 4));
283	if (is_shimm32(tmp, cmode, imm8) \|\|
284	is_soimm32(tmp, cmode, imm8)) {
285	break;
286	}
287	}
288	return i;
289	}
290
291	/* Return true if V is a valid 16-bit or 32-bit shifted immediate. */
292	static bool is_shimm1632(uint32_t v32, int cmode, int imm8)
293	{
294	if (v32 == deposit32(v32, 16, 16, v32)) {
295	return is_shimm16(v32, cmode, imm8);
296	} else {
297	return is_shimm32(v32, cmode, imm8);
298	}
299	}
300
301	static bool tcg_target_const_match(int64_t val, int ct,
302	TCGType type, TCGCond cond, int vece)
303	{
304	if (ct & TCG_CT_CONST) {
305	return 1;
306	}
307	if (type == TCG_TYPE_I32) {
308	val = (int32_t)val;
309	}
310
311	if (ct & TCG_CT_CONST_CMP) {
312	if (is_tst_cond(cond)) {
313	ct \|= TCG_CT_CONST_LIMM;
314	} else {
315	ct \|= TCG_CT_CONST_AIMM;
316	}
317	}
318
319	if ((ct & TCG_CT_CONST_AIMM) && (is_aimm(val) \|\| is_aimm(-val))) {
320	return 1;
321	}
322	if ((ct & TCG_CT_CONST_LIMM) && is_limm(val)) {
323	return 1;
324	}
325	if ((ct & TCG_CT_CONST_S8) && val == (int8_t)val) {
326	return 1;
327	}
328	if ((ct & TCG_CT_CONST_U8) && val == (uint8_t)val) {
329	return 1;
330	}
331	if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
332	return 1;
333	}
334	if ((ct & TCG_CT_CONST_MONE) && val == -1) {
335	return 1;
336	}
337
338	switch (ct & (TCG_CT_CONST_ORRI \| TCG_CT_CONST_ANDI)) {
339	case 0:
340	break;
341	case TCG_CT_CONST_ANDI:
342	val = ~val;
343	/* fallthru */
344	case TCG_CT_CONST_ORRI:
345	if (val == deposit64(val, 32, 32, val)) {
346	int cmode, imm8;
347	return is_shimm1632(val, &cmode, &imm8);
348	}
349	break;
350	default:
351	/* Both bits should not be set for the same insn. */
352	g_assert_not_reached();
353	}
354
355	return 0;
356	}
357
358	enum aarch64_cond_code {
359	COND_EQ = 0x0,
360	COND_NE = 0x1,
361	COND_CS = 0x2, /* Unsigned greater or equal */
362	COND_HS = COND_CS, /* ALIAS greater or equal */
363	COND_CC = 0x3, /* Unsigned less than */
364	COND_LO = COND_CC, /* ALIAS Lower */
365	COND_MI = 0x4, /* Negative */
366	COND_PL = 0x5, /* Zero or greater */
367	COND_VS = 0x6, /* Overflow */
368	COND_VC = 0x7, /* No overflow */
369	COND_HI = 0x8, /* Unsigned greater than */
370	COND_LS = 0x9, /* Unsigned less or equal */
371	COND_GE = 0xa,
372	COND_LT = 0xb,
373	COND_GT = 0xc,
374	COND_LE = 0xd,
375	COND_AL = 0xe,
376	COND_NV = 0xf, /* behaves like COND_AL here */
377	};
378
379	static const enum aarch64_cond_code tcg_cond_to_aarch64[] = {
380	[TCG_COND_EQ] = COND_EQ,
381	[TCG_COND_NE] = COND_NE,
382	[TCG_COND_LT] = COND_LT,
383	[TCG_COND_GE] = COND_GE,
384	[TCG_COND_LE] = COND_LE,
385	[TCG_COND_GT] = COND_GT,
386	/* unsigned */
387	[TCG_COND_LTU] = COND_LO,
388	[TCG_COND_GTU] = COND_HI,
389	[TCG_COND_GEU] = COND_HS,
390	[TCG_COND_LEU] = COND_LS,
391	/* bit test */
392	[TCG_COND_TSTEQ] = COND_EQ,
393	[TCG_COND_TSTNE] = COND_NE,
394	};
395
396	typedef enum {
397	LDST_ST = 0, /* store */
398	LDST_LD = 1, /* load */
399	LDST_LD_S_X = 2, /* load and sign-extend into Xt */
400	LDST_LD_S_W = 3, /* load and sign-extend into Wt */
401	} AArch64LdstType;
402
403	/* We encode the format of the insn into the beginning of the name, so that
404	we can have the preprocessor help "typecheck" the insn vs the output
405	function. Arm didn't provide us with nice names for the formats, so we
406	use the section number of the architecture reference manual in which the
407	instruction group is described. */
408	typedef enum {
409	/* Compare and branch (immediate). */
410	Icbz_CBZ = 0x34000000,
411	Icbz_CBNZ = 0x35000000,
412
413	/* Conditional branch (immediate). */
414	Ibcond_imm_B_C = 0x54000000,
415
416	/* Test and branch (immediate). */
417	Itbz_TBZ = 0x36000000,
418	Itbz_TBNZ = 0x37000000,
419
420	/* Unconditional branch (immediate). */
421	Ibranch_B = 0x14000000,
422	Ibranch_BL = 0x94000000,
423
424	/* Unconditional branch (register). */
425	Ibcond_reg_BR = 0xd61f0000,
426	Ibcond_reg_BLR = 0xd63f0000,
427	Ibcond_reg_RET = 0xd65f0000,
428
429	/* AdvSIMD load/store single structure. */
430	Isimd_loadrep_LD1R = 0x0d40c000,
431
432	/* Load literal for loading the address at pc-relative offset */
433	Ildlit_LDR = 0x58000000,
434	Ildlit_LDR_v64 = 0x5c000000,
435	Ildlit_LDR_v128 = 0x9c000000,
436
437	/* Load/store exclusive. */
438	Istxp_LDXP = 0xc8600000,
439	Istxp_STXP = 0xc8200000,
440
441	/* Load/store register. Described here as 3.3.12, but the helper
442	that emits them can transform to 3.3.10 or 3.3.13. */
443	Ildst_imm_STRB = 0x38000000 \| LDST_ST << 22 \| MO_8 << 30,
444	Ildst_imm_STRH = 0x38000000 \| LDST_ST << 22 \| MO_16 << 30,
445	Ildst_imm_STRW = 0x38000000 \| LDST_ST << 22 \| MO_32 << 30,
446	Ildst_imm_STRX = 0x38000000 \| LDST_ST << 22 \| MO_64 << 30,
447
448	Ildst_imm_LDRB = 0x38000000 \| LDST_LD << 22 \| MO_8 << 30,
449	Ildst_imm_LDRH = 0x38000000 \| LDST_LD << 22 \| MO_16 << 30,
450	Ildst_imm_LDRW = 0x38000000 \| LDST_LD << 22 \| MO_32 << 30,
451	Ildst_imm_LDRX = 0x38000000 \| LDST_LD << 22 \| MO_64 << 30,
452
453	Ildst_imm_LDRSBW = 0x38000000 \| LDST_LD_S_W << 22 \| MO_8 << 30,
454	Ildst_imm_LDRSHW = 0x38000000 \| LDST_LD_S_W << 22 \| MO_16 << 30,
455
456	Ildst_imm_LDRSBX = 0x38000000 \| LDST_LD_S_X << 22 \| MO_8 << 30,
457	Ildst_imm_LDRSHX = 0x38000000 \| LDST_LD_S_X << 22 \| MO_16 << 30,
458	Ildst_imm_LDRSWX = 0x38000000 \| LDST_LD_S_X << 22 \| MO_32 << 30,
459
460	Ildst_imm_LDRVS = 0x3c000000 \| LDST_LD << 22 \| MO_32 << 30,
461	Ildst_imm_STRVS = 0x3c000000 \| LDST_ST << 22 \| MO_32 << 30,
462
463	Ildst_imm_LDRVD = 0x3c000000 \| LDST_LD << 22 \| MO_64 << 30,
464	Ildst_imm_STRVD = 0x3c000000 \| LDST_ST << 22 \| MO_64 << 30,
465
466	Ildst_imm_LDRVQ = 0x3c000000 \| 3 << 22 \| 0 << 30,
467	Ildst_imm_STRVQ = 0x3c000000 \| 2 << 22 \| 0 << 30,
468
469	/* Additions to the ldst_imm format */
470	ldst_imm_to_reg = 0x00200800,
471	ldst_imm_to_uimm = 0x01000000,
472
473	/* Load/store register pair instructions. */
474	Ildstpair_LDP = 0x28400000,
475	Ildstpair_STP = 0x28000000,
476
477	/* Add/subtract immediate instructions. */
478	Iaddsub_imm_ADDI = 0x11000000,
479	Iaddsub_imm_ADDSI = 0x31000000,
480	Iaddsub_imm_SUBI = 0x51000000,
481	Iaddsub_imm_SUBSI = 0x71000000,
482
483	/* Min/max immediate instructions. */
484	Iminmax_imm_SMAXI = 0x11c00000,
485	Iminmax_imm_UMAXI = 0x11c40000,
486	Iminmax_imm_SMINI = 0x11c80000,
487	Iminmax_imm_UMINI = 0x11cc0000,
488
489	/* Bitfield instructions. */
490	Ibitfield_BFM = 0x33000000,
491	Ibitfield_SBFM = 0x13000000,
492	Ibitfield_UBFM = 0x53000000,
493
494	/* Extract instruction. */
495	Iextract_EXTR = 0x13800000,
496
497	/* Logical immediate instructions. */
498	Ilogic_imm_ANDI = 0x12000000,
499	Ilogic_imm_ORRI = 0x32000000,
500	Ilogic_imm_EORI = 0x52000000,
501	Ilogic_imm_ANDSI = 0x72000000,
502
503	/* Move wide immediate instructions. */
504	Imovw_MOVN = 0x12800000,
505	Imovw_MOVZ = 0x52800000,
506	Imovw_MOVK = 0x72800000,
507
508	/* PC relative addressing instructions. */
509	Ipcrel_ADR = 0x10000000,
510	Ipcrel_ADRP = 0x90000000,
511
512	/* Add/subtract extended register instructions. */
513	Iaddsub_ext_ADD = 0x0b200000,
514
515	/* Add/subtract shifted register instructions (without a shift). */
516	Iaddsub_shift_ADD = 0x0b000000,
517	Iaddsub_shift_ADDS = 0x2b000000,
518	Iaddsub_shift_SUB = 0x4b000000,
519	Iaddsub_shift_SUBS = 0x6b000000,
520
521	/* Add/subtract shifted register instructions (with a shift). */
522	Iaddsub_realshift_ADD_LSL = Iaddsub_shift_ADD,
523
524	/* Add/subtract with carry instructions. */
525	Irrr_sf_ADC = 0x1a000000,
526	Irrr_sf_ADCS = 0x3a000000,
527	Irrr_sf_SBC = 0x5a000000,
528	Irrr_sf_SBCS = 0x7a000000,
529
530	/* Conditional select instructions. */
531	Icsel_CSEL = 0x1a800000,
532	Icsel_CSINC = 0x1a800400,
533	Icsel_CSINV = 0x5a800000,
534	Icsel_CSNEG = 0x5a800400,
535
536	/* Data-processing (1 source) instructions. */
537	Irr_sf_CLZ = 0x5ac01000,
538	Irr_sf_CTZ = 0x5ac01800,
539	Irr_sf_CNT = 0x5ac01c00,
540	Irr_sf_RBIT = 0x5ac00000,
541	Irr_sf_REV = 0x5ac00000, /* + size << 10 */
542
543	/* Data-processing (2 source) instructions. */
544	Irrr_LSLV = 0x1ac02000,
545	Irrr_LSRV = 0x1ac02400,
546	Irrr_ASRV = 0x1ac02800,
547	Irrr_RORV = 0x1ac02c00,
548	Irrr_SMULH = 0x9b407c00,
549	Irrr_UMULH = 0x9bc07c00,
550	Irrr_UDIV = 0x1ac00800,
551	Irrr_SDIV = 0x1ac00c00,
552	Irrr_SMAX = 0x1ac00600,
553	Irrr_UMAX = 0x1ac00640,
554	Irrr_SMIN = 0x1ac00680,
555	Irrr_UMIN = 0x1ac006c0,
556
557	/* Data-processing (3 source) instructions. */
558	Irrrr_MADD = 0x1b000000,
559	Irrrr_MSUB = 0x1b008000,
560
561	/* Logical shifted register instructions (without a shift). */
562	Ilogic_shift_AND = 0x0a000000,
563	Ilogic_shift_BIC = 0x0a200000,
564	Ilogic_shift_ORR = 0x2a000000,
565	Ilogic_shift_ORN = 0x2a200000,
566	Ilogic_shift_EOR = 0x4a000000,
567	Ilogic_shift_EON = 0x4a200000,
568	Ilogic_shift_ANDS = 0x6a000000,
569
570	/* Logical shifted register instructions (with a shift). */
571	Iaddsub_realshift_AND_LSR = Ilogic_shift_AND \| (1 << 22),
572
573	/* AdvSIMD copy */
574	Isimd_copy_DUP = 0x0e000400,
575	Isimd_copy_INS = 0x4e001c00,
576	Isimd_copy_UMOV = 0x0e003c00,
577
578	/* AdvSIMD modified immediate */
579	Isimd_imm_MOVI = 0x0f000400,
580	Isimd_imm_MVNI = 0x2f000400,
581	Isimd_imm_BIC = 0x2f001400,
582	Isimd_imm_ORR = 0x0f001400,
583
584	/* AdvSIMD scalar shift by immediate */
585	Iq_shift_SSHR = 0x5f000400,
586	Iq_shift_SSRA = 0x5f001400,
587	Iq_shift_SHL = 0x5f005400,
588	Iq_shift_USHR = 0x7f000400,
589	Iq_shift_USRA = 0x7f001400,
590	Iq_shift_SLI = 0x7f005400,
591
592	/* AdvSIMD scalar three same */
593	Irrr_e_SQADD = 0x5e200c00,
594	Irrr_e_SQSUB = 0x5e202c00,
595	Irrr_e_CMGT = 0x5e203400,
596	Irrr_e_CMGE = 0x5e203c00,
597	Irrr_e_SSHL = 0x5e204400,
598	Irrr_e_ADD = 0x5e208400,
599	Irrr_e_CMTST = 0x5e208c00,
600	Irrr_e_UQADD = 0x7e200c00,
601	Irrr_e_UQSUB = 0x7e202c00,
602	Irrr_e_CMHI = 0x7e203400,
603	Irrr_e_CMHS = 0x7e203c00,
604	Irrr_e_USHL = 0x7e204400,
605	Irrr_e_SUB = 0x7e208400,
606	Irrr_e_CMEQ = 0x7e208c00,
607
608	/* AdvSIMD scalar two-reg misc */
609	Isimd_rr_CMGT0 = 0x5e208800,
610	Isimd_rr_CMEQ0 = 0x5e209800,
611	Isimd_rr_CMLT0 = 0x5e20a800,
612	Isimd_rr_ABS = 0x5e20b800,
613	Isimd_rr_CMGE0 = 0x7e208800,
614	Isimd_rr_CMLE0 = 0x7e209800,
615	Isimd_rr_NEG = 0x7e20b800,
616
617	/* AdvSIMD shift by immediate */
618	Isimd_shift_imm_SSHR = 0x0f000400,
619	Isimd_shift_imm_SSRA = 0x0f001400,
620	Isimd_shift_imm_SHL = 0x0f005400,
621	Isimd_shift_imm_SLI = 0x2f005400,
622	Isimd_shift_imm_USHR = 0x2f000400,
623	Isimd_shift_imm_USRA = 0x2f001400,
624
625	/* AdvSIMD three same. */
626	Iqrrr_e_ADD = 0x0e208400,
627	Iqrrr_e_AND = 0x0e201c00,
628	Iqrrr_e_BIC = 0x0e601c00,
629	Iqrrr_e_BIF = 0x2ee01c00,
630	Iqrrr_e_BIT = 0x2ea01c00,
631	Iqrrr_e_BSL = 0x2e601c00,
632	Iqrrr_e_EOR = 0x2e201c00,
633	Iqrrr_e_MUL = 0x0e209c00,
634	Iqrrr_e_ORR = 0x0ea01c00,
635	Iqrrr_e_ORN = 0x0ee01c00,
636	Iqrrr_e_SUB = 0x2e208400,
637	Iqrrr_e_CMGT = 0x0e203400,
638	Iqrrr_e_CMGE = 0x0e203c00,
639	Iqrrr_e_CMTST = 0x0e208c00,
640	Iqrrr_e_CMHI = 0x2e203400,
641	Iqrrr_e_CMHS = 0x2e203c00,
642	Iqrrr_e_CMEQ = 0x2e208c00,
643	Iqrrr_e_SMAX = 0x0e206400,
644	Iqrrr_e_SMIN = 0x0e206c00,
645	Iqrrr_e_SSHL = 0x0e204400,
646	Iqrrr_e_SQADD = 0x0e200c00,
647	Iqrrr_e_SQSUB = 0x0e202c00,
648	Iqrrr_e_UMAX = 0x2e206400,
649	Iqrrr_e_UMIN = 0x2e206c00,
650	Iqrrr_e_UQADD = 0x2e200c00,
651	Iqrrr_e_UQSUB = 0x2e202c00,
652	Iqrrr_e_USHL = 0x2e204400,
653
654	/* AdvSIMD two-reg misc. */
655	Iqrr_e_CMGT0 = 0x0e208800,
656	Iqrr_e_CMEQ0 = 0x0e209800,
657	Iqrr_e_CMLT0 = 0x0e20a800,
658	Iqrr_e_CMGE0 = 0x2e208800,
659	Iqrr_e_CMLE0 = 0x2e209800,
660	Iqrr_e_NOT = 0x2e205800,
661	Iqrr_e_ABS = 0x0e20b800,
662	Iqrr_e_NEG = 0x2e20b800,
663
664	/* System instructions. */
665	NOP = 0xd503201f,
666	DMB_ISH = 0xd50338bf,
667	DMB_LD = 0x00000100,
668	DMB_ST = 0x00000200,
669
670	BTI_C = 0xd503245f,
671	BTI_J = 0xd503249f,
672	BTI_JC = 0xd50324df,
673	} AArch64Insn;
674
675	static inline uint32_t tcg_in32(TCGContext *s)
676	{
677	uint32_t v = (uint32_t )s->code_ptr;
678	return v;
679	}
680
681	/* Emit an opcode with "type-checking" of the format. */
682	#define tcg_out_insn(S, FMT, OP, ...) \
683	glue(tcg_out_insn_,FMT)(S, glue(glue(glue(I,FMT),_),OP), ## __VA_ARGS__)
684
685	static void tcg_out_insn_simd_loadrep(TCGContext *s, AArch64Insn insn, bool q,
686	TCGReg rt, TCGReg rn, unsigned size)
687	{
688	tcg_out32(s, insn \| (rt & 0x1f) \| (rn << 5) \| (size << 10) \| (q << 30));
689	}
690
691	static void tcg_out_insn_ldlit(TCGContext *s, AArch64Insn insn,
692	int imm19, TCGReg rt)
693	{
694	tcg_out32(s, insn \| (imm19 & 0x7ffff) << 5 \| rt);
695	}
696
697	static void tcg_out_insn_stxp(TCGContext *s, AArch64Insn insn, TCGReg rs,
698	TCGReg rt, TCGReg rt2, TCGReg rn)
699	{
700	tcg_out32(s, insn \| rs << 16 \| rt2 << 10 \| rn << 5 \| rt);
701	}
702
703	static void tcg_out_insn_cbz(TCGContext *s, AArch64Insn insn, TCGType ext,
704	TCGReg rt, int imm19)
705	{
706	tcg_out32(s, insn \| ext << 31 \| (imm19 & 0x7ffff) << 5 \| rt);
707	}
708
709	static void tcg_out_insn_bcond_imm(TCGContext *s, AArch64Insn insn,
710	TCGCond c, int imm19)
711	{
712	tcg_out32(s, insn \| tcg_cond_to_aarch64[c] \| (imm19 & 0x7ffff) << 5);
713	}
714
715	static void tcg_out_insn_tbz(TCGContext *s, AArch64Insn insn,
716	TCGReg rt, int imm6, int imm14)
717	{
718	insn \|= (imm6 & 0x20) << (31 - 5);
719	insn \|= (imm6 & 0x1f) << 19;
720	tcg_out32(s, insn \| (imm14 & 0x3fff) << 5 \| rt);
721	}
722
723	static void tcg_out_insn_branch(TCGContext *s, AArch64Insn insn, int imm26)
724	{
725	tcg_out32(s, insn \| (imm26 & 0x03ffffff));
726	}
727
728	static void tcg_out_insn_bcond_reg(TCGContext *s, AArch64Insn insn, TCGReg rn)
729	{
730	tcg_out32(s, insn \| rn << 5);
731	}
732
733	static void tcg_out_insn_ldstpair(TCGContext *s, AArch64Insn insn,
734	TCGReg r1, TCGReg r2, TCGReg rn,
735	tcg_target_long ofs, bool pre, bool w)
736	{
737	insn \|= 1u << 31; /* ext */
738	insn \|= pre << 24;
739	insn \|= w << 23;
740
741	tcg_debug_assert(ofs >= -0x200 && ofs < 0x200 && (ofs & 7) == 0);
742	insn \|= (ofs & (0x7f << 3)) << (15 - 3);
743
744	tcg_out32(s, insn \| r2 << 10 \| rn << 5 \| r1);
745	}
746
747	static void tcg_out_insn_addsub_imm(TCGContext *s, AArch64Insn insn,
748	TCGType ext, TCGReg rd, TCGReg rn,
749	uint64_t aimm)
750	{
751	if (aimm > 0xfff) {
752	tcg_debug_assert((aimm & 0xfff) == 0);
753	aimm >>= 12;
754	tcg_debug_assert(aimm <= 0xfff);
755	aimm \|= 1 << 12; /* apply LSL 12 */
756	}
757	tcg_out32(s, insn \| ext << 31 \| aimm << 10 \| rn << 5 \| rd);
758	}
759
760	static void tcg_out_insn_minmax_imm(TCGContext *s, AArch64Insn insn,
761	TCGType ext, TCGReg rd, TCGReg rn,
762	uint8_t imm)
763	{
764	tcg_out32(s, insn \| ext << 31 \| imm << 10 \| rn << 5 \| rd);
765	}
766
767	/* This function can be used for both 3.4.2 (Bitfield) and 3.4.4
768	(Logical immediate). Both insn groups have N, IMMR and IMMS fields
769	that feed the DecodeBitMasks pseudo function. */
770	static void tcg_out_insn_bitfield(TCGContext *s, AArch64Insn insn, TCGType ext,
771	TCGReg rd, TCGReg rn, int n, int immr,
772	int imms)
773	{
774	tcg_out32(s, insn \| ext << 31 \| n << 22 \| immr << 16 \| imms << 10
775	\| rn << 5 \| rd);
776	}
777
778	#define tcg_out_insn_logic_imm tcg_out_insn_bitfield
779
780	static void tcg_out_insn_extract(TCGContext *s, AArch64Insn insn, TCGType ext,
781	TCGReg rd, TCGReg rn, TCGReg rm, int imms)
782	{
783	tcg_out32(s, insn \| ext << 31 \| ext << 22 \| rm << 16 \| imms << 10
784	\| rn << 5 \| rd);
785	}
786
787	/* This function is used for the Move (wide immediate) instruction group.
788	Note that SHIFT is a full shift count, not the 2 bit HW field. */
789	static void tcg_out_insn_movw(TCGContext *s, AArch64Insn insn, TCGType ext,
790	TCGReg rd, uint16_t half, unsigned shift)
791	{
792	tcg_debug_assert((shift & ~0x30) == 0);
793	tcg_out32(s, insn \| ext << 31 \| shift << (21 - 4) \| half << 5 \| rd);
794	}
795
796	static void tcg_out_insn_pcrel(TCGContext *s, AArch64Insn insn,
797	TCGReg rd, int64_t disp)
798	{
799	tcg_out32(s, insn \| (disp & 3) << 29 \| (disp & 0x1ffffc) << (5 - 2) \| rd);
800	}
801
802	static inline void tcg_out_insn_addsub_ext(TCGContext *s, AArch64Insn insn,
803	TCGType sf, TCGReg rd, TCGReg rn,
804	TCGReg rm, int opt, int imm3)
805	{
806	tcg_out32(s, insn \| sf << 31 \| rm << 16 \| opt << 13 \|
807	imm3 << 10 \| rn << 5 \| rd);
808	}
809
810	/* This function is for both 3.5.2 (Add/Subtract shifted register), for
811	the rare occasion when we actually want to supply a shift amount. */
812	static inline void tcg_out_insn_addsub_realshift(TCGContext *s,
813	AArch64Insn insn,
814	TCGType ext, TCGReg rd,
815	TCGReg rn, TCGReg rm,
816	int imm6)
817	{
818	tcg_out32(s, insn \| ext << 31 \| rm << 16 \| imm6 << 10 \| rn << 5 \| rd);
819	}
820
821	/* This function is for 3.5.2 (Add/subtract shifted register),
822	and 3.5.10 (Logical shifted register), for the vast majorty of cases
823	when we don't want to apply a shift. Thus it can also be used for
824	3.5.3 (Add/subtract with carry) and 3.5.8 (Data processing 2 source). */
825	static void tcg_out_insn_addsub_shift(TCGContext *s, AArch64Insn insn,
826	TCGType ext, TCGReg rd, TCGReg rn,
827	TCGReg rm)
828	{
829	tcg_out32(s, insn \| ext << 31 \| rm << 16 \| rn << 5 \| rd);
830	}
831
832	#define tcg_out_insn_rrr_sf tcg_out_insn_addsub_shift
833	#define tcg_out_insn_rrr tcg_out_insn_addsub_shift
834	#define tcg_out_insn_logic_shift tcg_out_insn_addsub_shift
835
836	static void tcg_out_insn_csel(TCGContext *s, AArch64Insn insn, TCGType ext,
837	TCGReg rd, TCGReg rn, TCGReg rm, TCGCond c)
838	{
839	tcg_out32(s, insn \| ext << 31 \| rm << 16 \| rn << 5 \| rd
840	\| tcg_cond_to_aarch64[c] << 12);
841	}
842
843	static void tcg_out_insn_rr_sf(TCGContext *s, AArch64Insn insn, TCGType ext,
844	TCGReg rd, TCGReg rn)
845	{
846	tcg_out32(s, insn \| ext << 31 \| rn << 5 \| rd);
847	}
848
849	static void tcg_out_insn_rrrr(TCGContext *s, AArch64Insn insn, TCGType ext,
850	TCGReg rd, TCGReg rn, TCGReg rm, TCGReg ra)
851	{
852	tcg_out32(s, insn \| ext << 31 \| rm << 16 \| ra << 10 \| rn << 5 \| rd);
853	}
854
855	static void tcg_out_insn_simd_copy(TCGContext *s, AArch64Insn insn, bool q,
856	TCGReg rd, TCGReg rn, int dst_idx, int src_idx)
857	{
858	/* Note that bit 11 set means general register input. Therefore
859	we can handle both register sets with one function. */
860	tcg_out32(s, insn \| q << 30 \| (dst_idx << 16) \| (src_idx << 11)
861	\| (rd & 0x1f) \| (~rn & 0x20) << 6 \| (rn & 0x1f) << 5);
862	}
863
864	static void tcg_out_insn_simd_imm(TCGContext *s, AArch64Insn insn, bool q,
865	TCGReg rd, bool op, int cmode, uint8_t imm8)
866	{
867	tcg_out32(s, insn \| q << 30 \| op << 29 \| cmode << 12 \| (rd & 0x1f)
868	\| (imm8 & 0xe0) << (16 - 5) \| (imm8 & 0x1f) << 5);
869	}
870
871	static void tcg_out_insn_q_shift(TCGContext *s, AArch64Insn insn,
872	TCGReg rd, TCGReg rn, unsigned immhb)
873	{
874	tcg_out32(s, insn \| immhb << 16 \| (rn & 0x1f) << 5 \| (rd & 0x1f));
875	}
876
877	static void tcg_out_insn_rrr_e(TCGContext *s, AArch64Insn insn,
878	unsigned size, TCGReg rd, TCGReg rn, TCGReg rm)
879	{
880	tcg_out32(s, insn \| (size << 22) \| (rm & 0x1f) << 16
881	\| (rn & 0x1f) << 5 \| (rd & 0x1f));
882	}
883
884	static void tcg_out_insn_simd_rr(TCGContext *s, AArch64Insn insn,
885	unsigned size, TCGReg rd, TCGReg rn)
886	{
887	tcg_out32(s, insn \| (size << 22) \| (rn & 0x1f) << 5 \| (rd & 0x1f));
888	}
889
890	static void tcg_out_insn_simd_shift_imm(TCGContext *s, AArch64Insn insn, bool q,
891	TCGReg rd, TCGReg rn, unsigned immhb)
892	{
893	tcg_out32(s, insn \| q << 30 \| immhb << 16
894	\| (rn & 0x1f) << 5 \| (rd & 0x1f));
895	}
896
897	static void tcg_out_insn_qrrr_e(TCGContext *s, AArch64Insn insn, bool q,
898	unsigned size, TCGReg rd, TCGReg rn, TCGReg rm)
899	{
900	tcg_out32(s, insn \| q << 30 \| (size << 22) \| (rm & 0x1f) << 16
901	\| (rn & 0x1f) << 5 \| (rd & 0x1f));
902	}
903
904	static void tcg_out_insn_qrr_e(TCGContext *s, AArch64Insn insn, bool q,
905	unsigned size, TCGReg rd, TCGReg rn)
906	{
907	tcg_out32(s, insn \| q << 30 \| (size << 22)
908	\| (rn & 0x1f) << 5 \| (rd & 0x1f));
909	}
910
911	static void tcg_out_insn_ldst_reg(TCGContext *s, AArch64Insn insn,
912	TCGReg rd, TCGReg base, TCGType ext,
913	TCGReg regoff)
914	{
915	/* Note the AArch64Insn constants above are for C3.3.12. Adjust. */
916	tcg_out32(s, insn \| ldst_imm_to_reg \| regoff << 16 \| 0x4000 \| ext << 13 \|
917	base << 5 \| (rd & 0x1f));
918	}
919
920	static void tcg_out_insn_ldst_imm(TCGContext *s, AArch64Insn insn,
921	TCGReg rd, TCGReg rn, intptr_t offset)
922	{
923	tcg_out32(s, insn \| (offset & 0x1ff) << 12 \| rn << 5 \| (rd & 0x1f));
924	}
925
926	static void tcg_out_insn_ldst_uimm(TCGContext *s, AArch64Insn insn,
927	TCGReg rd, TCGReg rn, uintptr_t scaled_uimm)
928	{
929	/* Note the AArch64Insn constants above are for C3.3.12. Adjust. */
930	tcg_out32(s, insn \| ldst_imm_to_uimm \| scaled_uimm << 10
931	\| rn << 5 \| (rd & 0x1f));
932	}
933
934	static void tcg_out_bti(TCGContext *s, AArch64Insn insn)
935	{
936	/*
937	* While BTI insns are nops on hosts without FEAT_BTI,
938	* there is no point in emitting them in that case either.
939	*/
940	if (cpuinfo & CPUINFO_BTI) {
941	tcg_out32(s, insn);
942	}
943	}
944
945	/* Register to register move using ORR (shifted register with no shift). */
946	static void tcg_out_movr(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rm)
947	{
948	tcg_out_insn(s, logic_shift, ORR, ext, rd, TCG_REG_XZR, rm);
949	}
950
951	/* Register to register move using ADDI (move to/from SP). */
952	static void tcg_out_movr_sp(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rn)
953	{
954	tcg_out_insn(s, addsub_imm, ADDI, ext, rd, rn, 0);
955	}
956
957	/* This function is used for the Logical (immediate) instruction group.
958	The value of LIMM must satisfy IS_LIMM. See the comment above about
959	only supporting simplified logical immediates. */
960	static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext,
961	TCGReg rd, TCGReg rn, uint64_t limm)
962	{
963	unsigned h, l, r, c;
964
965	tcg_debug_assert(is_limm(limm));
966
967	h = clz64(limm);
968	l = ctz64(limm);
969	if (l == 0) {
970	r = 0; /* form 0....01....1 */
971	c = ctz64(~limm) - 1;
972	if (h == 0) {
973	r = clz64(~limm); /* form 1..10..01..1 */
974	c += r;
975	}
976	} else {
977	r = 64 - l; /* form 1....10....0 or 0..01..10..0 */
978	c = r - h - 1;
979	}
980	if (ext == TCG_TYPE_I32) {
981	r &= 31;
982	c &= 31;
983	}
984
985	tcg_out_insn_logic_imm(s, insn, ext, rd, rn, ext, r, c);
986	}
987
988	static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
989	TCGReg rd, int64_t v64)
990	{
991	bool q = type == TCG_TYPE_V128;
992	int cmode, imm8, i;
993
994	/* Test all bytes equal first. */
995	if (vece == MO_8) {
996	imm8 = (uint8_t)v64;
997	tcg_out_insn(s, simd_imm, MOVI, q, rd, 0, 0xe, imm8);
998	return;
999	}
1000
1001	/*
1002	* Test all bytes 0x00 or 0xff second. This can match cases that
1003	* might otherwise take 2 or 3 insns for MO_16 or MO_32 below.
1004	*/
1005	for (i = imm8 = 0; i < 8; i++) {
1006	uint8_t byte = v64 >> (i * 8);
1007	if (byte == 0xff) {
1008	imm8 \|= 1 << i;
1009	} else if (byte != 0) {
1010	goto fail_bytes;
1011	}
1012	}
1013	tcg_out_insn(s, simd_imm, MOVI, q, rd, 1, 0xe, imm8);
1014	return;
1015	fail_bytes:
1016
1017	/*
1018	* Tests for various replications. For each element width, if we
1019	* cannot find an expansion there's no point checking a larger
1020	* width because we already know by replication it cannot match.
1021	*/
1022	if (vece == MO_16) {
1023	uint16_t v16 = v64;
1024
1025	if (is_shimm16(v16, &cmode, &imm8)) {
1026	tcg_out_insn(s, simd_imm, MOVI, q, rd, 0, cmode, imm8);
1027	return;
1028	}
1029	if (is_shimm16(~v16, &cmode, &imm8)) {
1030	tcg_out_insn(s, simd_imm, MVNI, q, rd, 0, cmode, imm8);
1031	return;
1032	}
1033
1034	/*
1035	* Otherwise, all remaining constants can be loaded in two insns:
1036	* rd = v16 & 0xff, rd \|= v16 & 0xff00.
1037	*/
1038	tcg_out_insn(s, simd_imm, MOVI, q, rd, 0, 0x8, v16 & 0xff);
1039	tcg_out_insn(s, simd_imm, ORR, q, rd, 0, 0xa, v16 >> 8);
1040	return;
1041	} else if (vece == MO_32) {
1042	uint32_t v32 = v64;
1043	uint32_t n32 = ~v32;
1044
1045	if (is_shimm32(v32, &cmode, &imm8) \|\|
1046	is_soimm32(v32, &cmode, &imm8) \|\|
1047	is_fimm32(v32, &cmode, &imm8)) {
1048	tcg_out_insn(s, simd_imm, MOVI, q, rd, 0, cmode, imm8);
1049	return;
1050	}
1051	if (is_shimm32(n32, &cmode, &imm8) \|\|
1052	is_soimm32(n32, &cmode, &imm8)) {
1053	tcg_out_insn(s, simd_imm, MVNI, q, rd, 0, cmode, imm8);
1054	return;
1055	}
1056
1057	/*
1058	* Restrict the set of constants to those we can load with
1059	* two instructions. Others we load from the pool.
1060	*/
1061	i = is_shimm32_pair(v32, &cmode, &imm8);
1062	if (i) {
1063	tcg_out_insn(s, simd_imm, MOVI, q, rd, 0, cmode, imm8);
1064	tcg_out_insn(s, simd_imm, ORR, q, rd, 0, i,
1065	extract32(v32, i * 4, 8));
1066	return;
1067	}
1068	i = is_shimm32_pair(n32, &cmode, &imm8);
1069	if (i) {
1070	tcg_out_insn(s, simd_imm, MVNI, q, rd, 0, cmode, imm8);
1071	tcg_out_insn(s, simd_imm, BIC, q, rd, 0, i,
1072	extract32(n32, i * 4, 8));
1073	return;
1074	}
1075	} else if (is_fimm64(v64, &cmode, &imm8)) {
1076	tcg_out_insn(s, simd_imm, MOVI, q, rd, 1, cmode, imm8);
1077	return;
1078	}
1079
1080	/*
1081	* As a last resort, load from the constant pool. Sadly there
1082	* is no LD1R (literal), so store the full 16-byte vector.
1083	*/
1084	if (type == TCG_TYPE_V128) {
1085	new_pool_l2(s, R_AARCH64_CONDBR19, s->code_ptr, 0, v64, v64);
1086	tcg_out_insn(s, ldlit, LDR_v128, 0, rd);
1087	} else {
1088	new_pool_label(s, v64, R_AARCH64_CONDBR19, s->code_ptr, 0);
1089	tcg_out_insn(s, ldlit, LDR_v64, 0, rd);
1090	}
1091	}
1092
1093	static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
1094	TCGReg rd, TCGReg rs)
1095	{
1096	int is_q = type - TCG_TYPE_V64;
1097	tcg_out_insn(s, simd_copy, DUP, is_q, rd, rs, 1 << vece, 0);
1098	return true;
1099	}
1100
1101	static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
1102	TCGReg r, TCGReg base, intptr_t offset)
1103	{
1104	TCGReg temp = TCG_REG_TMP0;
1105
1106	if (offset < -0xffffff \|\| offset > 0xffffff) {
1107	tcg_out_movi(s, TCG_TYPE_PTR, temp, offset);
1108	tcg_out_insn(s, addsub_shift, ADD, 1, temp, temp, base);
1109	base = temp;
1110	} else {
1111	AArch64Insn add_insn = Iaddsub_imm_ADDI;
1112
1113	if (offset < 0) {
1114	add_insn = Iaddsub_imm_SUBI;
1115	offset = -offset;
1116	}
1117	if (offset & 0xfff000) {
1118	tcg_out_insn_addsub_imm(s, add_insn, 1, temp, base,
1119	offset & 0xfff000);
1120	base = temp;
1121	}
1122	if (offset & 0xfff) {
1123	tcg_out_insn_addsub_imm(s, add_insn, 1, temp, base, offset & 0xfff);
1124	base = temp;
1125	}
1126	}
1127	tcg_out_insn(s, simd_loadrep, LD1R, type == TCG_TYPE_V128, r, base, vece);
1128	return true;
1129	}
1130
1131	static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
1132	tcg_target_long value)
1133	{
1134	tcg_target_long svalue = value;
1135	tcg_target_long ivalue = ~value;
1136	tcg_target_long t0, t1, t2;
1137	int s0, s1;
1138	AArch64Insn opc;
1139
1140	switch (type) {
1141	case TCG_TYPE_I32:
1142	case TCG_TYPE_I64:
1143	tcg_debug_assert(rd < 32);
1144	break;
1145	default:
1146	g_assert_not_reached();
1147	}
1148
1149	/* For 32-bit values, discard potential garbage in value. For 64-bit
1150	values within [231, 232-1], we can create smaller sequences by
1151	interpreting this as a negative 32-bit number, while ensuring that
1152	the high 32 bits are cleared by setting SF=0. */
1153	if (type == TCG_TYPE_I32 \|\| (value & ~0xffffffffull) == 0) {
1154	svalue = (int32_t)value;
1155	value = (uint32_t)value;
1156	ivalue = (uint32_t)ivalue;
1157	type = TCG_TYPE_I32;
1158	}
1159
1160	/* Speed things up by handling the common case of small positive
1161	and negative values specially. */
1162	if ((value & ~0xffffull) == 0) {
1163	tcg_out_insn(s, movw, MOVZ, type, rd, value, 0);
1164	return;
1165	} else if ((ivalue & ~0xffffull) == 0) {
1166	tcg_out_insn(s, movw, MOVN, type, rd, ivalue, 0);
1167	return;
1168	}
1169
1170	/* Check for bitfield immediates. For the benefit of 32-bit quantities,
1171	use the sign-extended value. That lets us match rotated values such
1172	as 0xff0000ff with the same 64-bit logic matching 0xffffffffff0000ff. */
1173	if (is_limm(svalue)) {
1174	tcg_out_logicali(s, Ilogic_imm_ORRI, type, rd, TCG_REG_XZR, svalue);
1175	return;
1176	}
1177
1178	/* Look for host pointer values within 4G of the PC. This happens
1179	often when loading pointers to QEMU's own data structures. */
1180	if (type == TCG_TYPE_I64) {
1181	intptr_t src_rx = (intptr_t)tcg_splitwx_to_rx(s->code_ptr);
1182	tcg_target_long disp = value - src_rx;
1183	if (disp == sextract64(disp, 0, 21)) {
1184	tcg_out_insn(s, pcrel, ADR, rd, disp);
1185	return;
1186	}
1187	disp = (value >> 12) - (src_rx >> 12);
1188	if (disp == sextract64(disp, 0, 21)) {
1189	tcg_out_insn(s, pcrel, ADRP, rd, disp);
1190	if (value & 0xfff) {
1191	tcg_out_insn(s, addsub_imm, ADDI, type, rd, rd, value & 0xfff);
1192	}
1193	return;
1194	}
1195	}
1196
1197	/* Would it take fewer insns to begin with MOVN? */
1198	if (ctpop64(value) >= 32) {
1199	t0 = ivalue;
1200	opc = Imovw_MOVN;
1201	} else {
1202	t0 = value;
1203	opc = Imovw_MOVZ;
1204	}
1205	s0 = ctz64(t0) & (63 & -16);
1206	t1 = t0 & ~(0xffffull << s0);
1207	s1 = ctz64(t1) & (63 & -16);
1208	t2 = t1 & ~(0xffffull << s1);
1209	if (t2 == 0) {
1210	tcg_out_insn_movw(s, opc, type, rd, t0 >> s0, s0);
1211	if (t1 != 0) {
1212	tcg_out_insn(s, movw, MOVK, type, rd, value >> s1, s1);
1213	}
1214	return;
1215	}
1216
1217	/* For more than 2 insns, dump it into the constant pool. */
1218	new_pool_label(s, value, R_AARCH64_CONDBR19, s->code_ptr, 0);
1219	tcg_out_insn(s, ldlit, LDR, 0, rd);
1220	}
1221
1222	static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
1223	{
1224	return false;
1225	}
1226
1227	static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
1228	tcg_target_long imm)
1229	{
1230	/* This function is only used for passing structs by reference. */
1231	g_assert_not_reached();
1232	}
1233
1234	static void tcg_out_ldst(TCGContext *s, AArch64Insn insn, TCGReg rd,
1235	TCGReg rn, intptr_t offset, int lgsize)
1236	{
1237	/* If the offset is naturally aligned and in range, then we can
1238	use the scaled uimm12 encoding */
1239	if (offset >= 0 && !(offset & ((1 << lgsize) - 1))) {
1240	uintptr_t scaled_uimm = offset >> lgsize;
1241	if (scaled_uimm <= 0xfff) {
1242	tcg_out_insn_ldst_uimm(s, insn, rd, rn, scaled_uimm);
1243	return;
1244	}
1245	}
1246
1247	/* Small signed offsets can use the unscaled encoding. */
1248	if (offset >= -256 && offset < 256) {
1249	tcg_out_insn_ldst_imm(s, insn, rd, rn, offset);
1250	return;
1251	}
1252
1253	/* Worst-case scenario, move offset to temp register, use reg offset. */
1254	tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP0, offset);
1255	tcg_out_insn_ldst_reg(s, insn, rd, rn, TCG_TYPE_I64, TCG_REG_TMP0);
1256	}
1257
1258	static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
1259	{
1260	if (ret == arg) {
1261	return true;
1262	}
1263	switch (type) {
1264	case TCG_TYPE_I32:
1265	case TCG_TYPE_I64:
1266	if (ret < 32 && arg < 32) {
1267	tcg_out_movr(s, type, ret, arg);
1268	break;
1269	} else if (ret < 32) {
1270	tcg_out_insn(s, simd_copy, UMOV, type, ret, arg, 0, 0);
1271	break;
1272	} else if (arg < 32) {
1273	tcg_out_insn(s, simd_copy, INS, 0, ret, arg, 4 << type, 0);
1274	break;
1275	}
1276	/* FALLTHRU */
1277
1278	case TCG_TYPE_V64:
1279	tcg_debug_assert(ret >= 32 && arg >= 32);
1280	tcg_out_insn(s, qrrr_e, ORR, 0, 0, ret, arg, arg);
1281	break;
1282	case TCG_TYPE_V128:
1283	tcg_debug_assert(ret >= 32 && arg >= 32);
1284	tcg_out_insn(s, qrrr_e, ORR, 1, 0, ret, arg, arg);
1285	break;
1286
1287	default:
1288	g_assert_not_reached();
1289	}
1290	return true;
1291	}
1292
1293	static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
1294	TCGReg base, intptr_t ofs)
1295	{
1296	AArch64Insn insn;
1297	int lgsz;
1298
1299	switch (type) {
1300	case TCG_TYPE_I32:
1301	insn = (ret < 32 ? Ildst_imm_LDRW : Ildst_imm_LDRVS);
1302	lgsz = 2;
1303	break;
1304	case TCG_TYPE_I64:
1305	insn = (ret < 32 ? Ildst_imm_LDRX : Ildst_imm_LDRVD);
1306	lgsz = 3;
1307	break;
1308	case TCG_TYPE_V64:
1309	insn = Ildst_imm_LDRVD;
1310	lgsz = 3;
1311	break;
1312	case TCG_TYPE_V128:
1313	insn = Ildst_imm_LDRVQ;
1314	lgsz = 4;
1315	break;
1316	default:
1317	g_assert_not_reached();
1318	}
1319	tcg_out_ldst(s, insn, ret, base, ofs, lgsz);
1320	}
1321
1322	static void tcg_out_st(TCGContext *s, TCGType type, TCGReg src,
1323	TCGReg base, intptr_t ofs)
1324	{
1325	AArch64Insn insn;
1326	int lgsz;
1327
1328	switch (type) {
1329	case TCG_TYPE_I32:
1330	insn = (src < 32 ? Ildst_imm_STRW : Ildst_imm_STRVS);
1331	lgsz = 2;
1332	break;
1333	case TCG_TYPE_I64:
1334	insn = (src < 32 ? Ildst_imm_STRX : Ildst_imm_STRVD);
1335	lgsz = 3;
1336	break;
1337	case TCG_TYPE_V64:
1338	insn = Ildst_imm_STRVD;
1339	lgsz = 3;
1340	break;
1341	case TCG_TYPE_V128:
1342	insn = Ildst_imm_STRVQ;
1343	lgsz = 4;
1344	break;
1345	default:
1346	g_assert_not_reached();
1347	}
1348	tcg_out_ldst(s, insn, src, base, ofs, lgsz);
1349	}
1350
1351	static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1352	TCGReg base, intptr_t ofs)
1353	{
1354	if (type <= TCG_TYPE_I64 && val == 0) {
1355	tcg_out_st(s, type, TCG_REG_XZR, base, ofs);
1356	return true;
1357	}
1358	return false;
1359	}
1360
1361	static inline void tcg_out_bfm(TCGContext *s, TCGType ext, TCGReg rd,
1362	TCGReg rn, unsigned int a, unsigned int b)
1363	{
1364	tcg_out_insn(s, bitfield, BFM, ext, rd, rn, ext, a, b);
1365	}
1366
1367	static inline void tcg_out_ubfm(TCGContext *s, TCGType ext, TCGReg rd,
1368	TCGReg rn, unsigned int a, unsigned int b)
1369	{
1370	tcg_out_insn(s, bitfield, UBFM, ext, rd, rn, ext, a, b);
1371	}
1372
1373	static inline void tcg_out_sbfm(TCGContext *s, TCGType ext, TCGReg rd,
1374	TCGReg rn, unsigned int a, unsigned int b)
1375	{
1376	tcg_out_insn(s, bitfield, SBFM, ext, rd, rn, ext, a, b);
1377	}
1378
1379	static inline void tcg_out_extr(TCGContext *s, TCGType ext, TCGReg rd,
1380	TCGReg rn, TCGReg rm, unsigned int a)
1381	{
1382	tcg_out_insn(s, extract, EXTR, ext, rd, rn, rm, a);
1383	}
1384
1385	static void tgen_cmp(TCGContext *s, TCGType ext, TCGCond cond,
1386	TCGReg a, TCGReg b)
1387	{
1388	if (is_tst_cond(cond)) {
1389	tcg_out_insn(s, logic_shift, ANDS, ext, TCG_REG_XZR, a, b);
1390	} else {
1391	tcg_out_insn(s, addsub_shift, SUBS, ext, TCG_REG_XZR, a, b);
1392	}
1393	}
1394
1395	static void tgen_cmpi(TCGContext *s, TCGType ext, TCGCond cond,
1396	TCGReg a, tcg_target_long b)
1397	{
1398	if (is_tst_cond(cond)) {
1399	tcg_out_logicali(s, Ilogic_imm_ANDSI, ext, TCG_REG_XZR, a, b);
1400	} else if (b >= 0) {
1401	tcg_debug_assert(is_aimm(b));
1402	tcg_out_insn(s, addsub_imm, SUBSI, ext, TCG_REG_XZR, a, b);
1403	} else {
1404	tcg_debug_assert(is_aimm(-b));
1405	tcg_out_insn(s, addsub_imm, ADDSI, ext, TCG_REG_XZR, a, -b);
1406	}
1407	}
1408
1409	static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGCond cond, TCGReg a,
1410	tcg_target_long b, bool const_b)
1411	{
1412	if (const_b) {
1413	tgen_cmpi(s, ext, cond, a, b);
1414	} else {
1415	tgen_cmp(s, ext, cond, a, b);
1416	}
1417	}
1418
1419	static void tcg_out_goto(TCGContext s, const tcg_insn_unit target)
1420	{
1421	ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2;
1422	tcg_debug_assert(offset == sextract64(offset, 0, 26));
1423	tcg_out_insn(s, branch, B, offset);
1424	}
1425
1426	static void tcg_out_call_int(TCGContext s, const tcg_insn_unit target)
1427	{
1428	ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2;
1429	if (offset == sextract64(offset, 0, 26)) {
1430	tcg_out_insn(s, branch, BL, offset);
1431	} else {
1432	tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP0, (intptr_t)target);
1433	tcg_out_insn(s, bcond_reg, BLR, TCG_REG_TMP0);
1434	}
1435	}
1436
1437	static void tcg_out_call(TCGContext s, const tcg_insn_unit target,
1438	const TCGHelperInfo *info)
1439	{
1440	tcg_out_call_int(s, target);
1441	}
1442
1443	static void tcg_out_br(TCGContext s, TCGLabel l)
1444	{
1445	if (!l->has_value) {
1446	tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, l, 0);
1447	tcg_out_insn(s, branch, B, 0);
1448	} else {
1449	tcg_out_goto(s, l->u.value_ptr);
1450	}
1451	}
1452
1453	static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
1454	TCGReg a, TCGReg b, TCGLabel *l)
1455	{
1456	tgen_cmp(s, type, c, a, b);
1457	tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0);
1458	tcg_out_insn(s, bcond_imm, B_C, c, 0);
1459	}
1460
1461	static void tgen_brcondi(TCGContext *s, TCGType ext, TCGCond c,
1462	TCGReg a, tcg_target_long b, TCGLabel *l)
1463	{
1464	int tbit = -1;
1465	bool need_cmp = true;
1466
1467	switch (c) {
1468	case TCG_COND_EQ:
1469	case TCG_COND_NE:
1470	/* cmp xN,0; b.ne L -> cbnz xN,L */
1471	if (b == 0) {
1472	need_cmp = false;
1473	}
1474	break;
1475	case TCG_COND_LT:
1476	case TCG_COND_GE:
1477	/* cmp xN,0; b.mi L -> tbnz xN,63,L */
1478	if (b == 0) {
1479	c = (c == TCG_COND_LT ? TCG_COND_TSTNE : TCG_COND_TSTEQ);
1480	tbit = ext ? 63 : 31;
1481	need_cmp = false;
1482	}
1483	break;
1484	case TCG_COND_TSTEQ:
1485	case TCG_COND_TSTNE:
1486	/* tst xN,0xffffffff; b.ne L -> cbnz wN,L */
1487	if (b == UINT32_MAX) {
1488	c = tcg_tst_eqne_cond(c);
1489	ext = TCG_TYPE_I32;
1490	need_cmp = false;
1491	break;
1492	}
1493	/* tst xN,1<<B; b.ne L -> tbnz xN,B,L */
1494	if (is_power_of_2(b)) {
1495	tbit = ctz64(b);
1496	need_cmp = false;
1497	}
1498	break;
1499	default:
1500	break;
1501	}
1502
1503	if (need_cmp) {
1504	tgen_cmpi(s, ext, c, a, b);
1505	tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0);
1506	tcg_out_insn(s, bcond_imm, B_C, c, 0);
1507	return;
1508	}
1509
1510	if (tbit >= 0) {
1511	tcg_out_reloc(s, s->code_ptr, R_AARCH64_TSTBR14, l, 0);
1512	switch (c) {
1513	case TCG_COND_TSTEQ:
1514	tcg_out_insn(s, tbz, TBZ, a, tbit, 0);
1515	break;
1516	case TCG_COND_TSTNE:
1517	tcg_out_insn(s, tbz, TBNZ, a, tbit, 0);
1518	break;
1519	default:
1520	g_assert_not_reached();
1521	}
1522	} else {
1523	tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0);
1524	switch (c) {
1525	case TCG_COND_EQ:
1526	tcg_out_insn(s, cbz, CBZ, ext, a, 0);
1527	break;
1528	case TCG_COND_NE:
1529	tcg_out_insn(s, cbz, CBNZ, ext, a, 0);
1530	break;
1531	default:
1532	g_assert_not_reached();
1533	}
1534	}
1535	}
1536
1537	static const TCGOutOpBrcond outop_brcond = {
1538	.base.static_constraint = C_O0_I2(r, rC),
1539	.out_rr = tgen_brcond,
1540	.out_ri = tgen_brcondi,
1541	};
1542
1543	static inline void tcg_out_rev(TCGContext *s, int ext, MemOp s_bits,
1544	TCGReg rd, TCGReg rn)
1545	{
1546	/* REV, REV16, REV32 */
1547	tcg_out_insn_rr_sf(s, Irr_sf_REV \| (s_bits << 10), ext, rd, rn);
1548	}
1549
1550	static inline void tcg_out_sxt(TCGContext *s, TCGType ext, MemOp s_bits,
1551	TCGReg rd, TCGReg rn)
1552	{
1553	/* Using ALIASes SXTB, SXTH, SXTW, of SBFM Xd, Xn, #0, #7\|15\|31 */
1554	int bits = (8 << s_bits) - 1;
1555	tcg_out_sbfm(s, ext, rd, rn, 0, bits);
1556	}
1557
1558	static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
1559	{
1560	tcg_out_sxt(s, type, MO_8, rd, rn);
1561	}
1562
1563	static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rn)
1564	{
1565	tcg_out_sxt(s, type, MO_16, rd, rn);
1566	}
1567
1568	static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rn)
1569	{
1570	tcg_out_sxt(s, TCG_TYPE_I64, MO_32, rd, rn);
1571	}
1572
1573	static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
1574	{
1575	tcg_out_ext32s(s, rd, rn);
1576	}
1577
1578	static inline void tcg_out_uxt(TCGContext *s, MemOp s_bits,
1579	TCGReg rd, TCGReg rn)
1580	{
1581	/* Using ALIASes UXTB, UXTH of UBFM Wd, Wn, #0, #7\|15 */
1582	int bits = (8 << s_bits) - 1;
1583	tcg_out_ubfm(s, 0, rd, rn, 0, bits);
1584	}
1585
1586	static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rn)
1587	{
1588	tcg_out_uxt(s, MO_8, rd, rn);
1589	}
1590
1591	static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rn)
1592	{
1593	tcg_out_uxt(s, MO_16, rd, rn);
1594	}
1595
1596	static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rn)
1597	{
1598	tcg_out_movr(s, TCG_TYPE_I32, rd, rn);
1599	}
1600
1601	static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rn)
1602	{
1603	tcg_out_ext32u(s, rd, rn);
1604	}
1605
1606	static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn)
1607	{
1608	tcg_out_mov(s, TCG_TYPE_I32, rd, rn);
1609	}
1610
1611	static void tcg_out_mb(TCGContext *s, unsigned a0)
1612	{
1613	static const uint32_t sync[] = {
1614	[0 ... TCG_MO_ALL] = DMB_ISH \| DMB_LD \| DMB_ST,
1615	[TCG_MO_ST_ST] = DMB_ISH \| DMB_ST,
1616	[TCG_MO_LD_LD] = DMB_ISH \| DMB_LD,
1617	[TCG_MO_LD_ST] = DMB_ISH \| DMB_LD,
1618	[TCG_MO_LD_ST \| TCG_MO_LD_LD] = DMB_ISH \| DMB_LD,
1619	};
1620	tcg_out32(s, sync[a0 & TCG_MO_ALL]);
1621	}
1622
1623	typedef struct {
1624	TCGReg base;
1625	TCGReg index;
1626	TCGType index_ext;
1627	TCGAtomAlign aa;
1628	} HostAddress;
1629
1630	bool tcg_target_has_memory_bswap(MemOp memop)
1631	{
1632	return false;
1633	}
1634
1635	static const TCGLdstHelperParam ldst_helper_param = {
1636	.ntmp = 1, .tmp = { TCG_REG_TMP0 }
1637	};
1638
1639	static bool tcg_out_qemu_ld_slow_path(TCGContext s, TCGLabelQemuLdst lb)
1640	{
1641	MemOp opc = get_memop(lb->oi);
1642
1643	if (!reloc_pc19(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
1644	return false;
1645	}
1646
1647	tcg_out_ld_helper_args(s, lb, &ldst_helper_param);
1648	tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE]);
1649	tcg_out_ld_helper_ret(s, lb, false, &ldst_helper_param);
1650	tcg_out_goto(s, lb->raddr);
1651	return true;
1652	}
1653
1654	static bool tcg_out_qemu_st_slow_path(TCGContext s, TCGLabelQemuLdst lb)
1655	{
1656	MemOp opc = get_memop(lb->oi);
1657
1658	if (!reloc_pc19(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
1659	return false;
1660	}
1661
1662	tcg_out_st_helper_args(s, lb, &ldst_helper_param);
1663	tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE]);
1664	tcg_out_goto(s, lb->raddr);
1665	return true;
1666	}
1667
1668	/* We expect to use a 7-bit scaled negative offset from ENV. */
1669	#define MIN_TLB_MASK_TABLE_OFS -512
1670
1671	/*
1672	* For system-mode, perform the TLB load and compare.
1673	* For user-mode, perform any required alignment tests.
1674	* In both cases, return a TCGLabelQemuLdst structure if the slow path
1675	* is required and fill in @h with the host address for the fast path.
1676	*/
1677	static TCGLabelQemuLdst prepare_host_addr(TCGContext s, HostAddress *h,
1678	TCGReg addr_reg, MemOpIdx oi,
1679	bool is_ld)
1680	{
1681	TCGType addr_type = s->addr_type;
1682	TCGLabelQemuLdst *ldst = NULL;
1683	MemOp opc = get_memop(oi);
1684	MemOp s_bits = opc & MO_SIZE;
1685	unsigned a_mask;
1686
1687	h->aa = atom_and_align_for_opc(s, opc,
1688	have_lse2 ? MO_ATOM_WITHIN16
1689	: MO_ATOM_IFALIGN,
1690	s_bits == MO_128);
1691	a_mask = (1 << h->aa.align) - 1;
1692
1693	if (tcg_use_softmmu) {
1694	unsigned s_mask = (1u << s_bits) - 1;
1695	unsigned mem_index = get_mmuidx(oi);
1696	TCGReg addr_adj;
1697	uint64_t compare_mask;
1698
1699	ldst = new_ldst_label(s);
1700	ldst->is_ld = is_ld;
1701	ldst->oi = oi;
1702	ldst->addr_reg = addr_reg;
1703
1704	/* Load CPUTLBDescFast.{mask,table} into {tmp0,tmp1}. */
1705	QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0);
1706	QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 8);
1707	tcg_out_insn(s, ldstpair, LDP, TCG_REG_TMP0, TCG_REG_TMP1, TCG_AREG0,
1708	tlb_mask_table_ofs(s, mem_index), 1, 0);
1709
1710	/* Extract the TLB index from the address into X0. */
1711	tcg_out_insn(s, addsub_realshift, AND_LSR, TCG_TYPE_I64,
1712	TCG_REG_TMP0, TCG_REG_TMP0, addr_reg,
1713	TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1714
1715	/* Add the tlb_table pointer, forming the CPUTLBEntry address. */
1716	tcg_out_insn(s, addsub_shift, ADD, 1, TCG_REG_TMP1, TCG_REG_TMP1,
1717	TCG_REG_TMP0);
1718
1719	/* Load the tlb comparator into TMP0, and the fast path addend. */
1720	QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN);
1721	tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP1,
1722	is_ld ? offsetof(CPUTLBEntry, addr_read)
1723	: offsetof(CPUTLBEntry, addr_write));
1724	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1,
1725	offsetof(CPUTLBEntry, addend));
1726
1727	/*
1728	* For aligned accesses, we check the first byte and include
1729	* the alignment bits within the address. For unaligned access,
1730	* we check that we don't cross pages using the address of the
1731	* last byte of the access.
1732	*/
1733	if (a_mask >= s_mask) {
1734	addr_adj = addr_reg;
1735	} else {
1736	addr_adj = TCG_REG_TMP2;
1737	tcg_out_insn(s, addsub_imm, ADDI, addr_type,
1738	addr_adj, addr_reg, s_mask - a_mask);
1739	}
1740	compare_mask = (uint64_t)TARGET_PAGE_MASK \| a_mask;
1741
1742	/* Store the page mask part of the address into TMP2. */
1743	tcg_out_logicali(s, Ilogic_imm_ANDI, addr_type, TCG_REG_TMP2,
1744	addr_adj, compare_mask);
1745
1746	/* Perform the address comparison. */
1747	tcg_out_cmp(s, addr_type, TCG_COND_NE, TCG_REG_TMP0, TCG_REG_TMP2, 0);
1748
1749	/* If not equal, we jump to the slow path. */
1750	ldst->label_ptr[0] = s->code_ptr;
1751	tcg_out_insn(s, bcond_imm, B_C, TCG_COND_NE, 0);
1752
1753	h->base = TCG_REG_TMP1;
1754	h->index = addr_reg;
1755	h->index_ext = addr_type;
1756	} else {
1757	if (a_mask) {
1758	ldst = new_ldst_label(s);
1759
1760	ldst->is_ld = is_ld;
1761	ldst->oi = oi;
1762	ldst->addr_reg = addr_reg;
1763
1764	/* tst addr, #mask */
1765	tcg_out_logicali(s, Ilogic_imm_ANDSI, 0, TCG_REG_XZR, addr_reg,
1766	a_mask);
1767
1768	/* b.ne slow_path */
1769	ldst->label_ptr[0] = s->code_ptr;
1770	tcg_out_insn(s, bcond_imm, B_C, TCG_COND_NE, 0);
1771	}
1772
1773	if (guest_base \|\| addr_type == TCG_TYPE_I32) {
1774	h->base = TCG_REG_GUEST_BASE;
1775	h->index = addr_reg;
1776	h->index_ext = addr_type;
1777	} else {
1778	h->base = addr_reg;
1779	h->index = TCG_REG_XZR;
1780	h->index_ext = TCG_TYPE_I64;
1781	}
1782	}
1783
1784	return ldst;
1785	}
1786
1787	static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext,
1788	TCGReg data_r, HostAddress h)
1789	{
1790	switch (memop & MO_SSIZE) {
1791	case MO_UB:
1792	tcg_out_insn_ldst_reg(s, Ildst_imm_LDRB, data_r, h.base,
1793	h.index_ext, h.index);
1794	break;
1795	case MO_SB:
1796	tcg_out_insn_ldst_reg(s, ext ? Ildst_imm_LDRSBX : Ildst_imm_LDRSBW,
1797	data_r, h.base, h.index_ext, h.index);
1798	break;
1799	case MO_UW:
1800	tcg_out_insn_ldst_reg(s, Ildst_imm_LDRH, data_r, h.base, h.index_ext,
1801	h.index);
1802	break;
1803	case MO_SW:
1804	tcg_out_insn_ldst_reg(s, ext ? Ildst_imm_LDRSHX : Ildst_imm_LDRSHW,
1805	data_r, h.base, h.index_ext, h.index);
1806	break;
1807	case MO_UL:
1808	tcg_out_insn_ldst_reg(s, Ildst_imm_LDRW, data_r, h.base, h.index_ext,
1809	h.index);
1810	break;
1811	case MO_SL:
1812	tcg_out_insn_ldst_reg(s, Ildst_imm_LDRSWX, data_r, h.base, h.index_ext,
1813	h.index);
1814	break;
1815	case MO_UQ:
1816	tcg_out_insn_ldst_reg(s, Ildst_imm_LDRX, data_r, h.base, h.index_ext,
1817	h.index);
1818	break;
1819	default:
1820	g_assert_not_reached();
1821	}
1822	}
1823
1824	static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop,
1825	TCGReg data_r, HostAddress h)
1826	{
1827	switch (memop & MO_SIZE) {
1828	case MO_8:
1829	tcg_out_insn_ldst_reg(s, Ildst_imm_STRB, data_r, h.base,
1830	h.index_ext, h.index);
1831	break;
1832	case MO_16:
1833	tcg_out_insn_ldst_reg(s, Ildst_imm_STRH, data_r, h.base,
1834	h.index_ext, h.index);
1835	break;
1836	case MO_32:
1837	tcg_out_insn_ldst_reg(s, Ildst_imm_STRW, data_r, h.base,
1838	h.index_ext, h.index);
1839	break;
1840	case MO_64:
1841	tcg_out_insn_ldst_reg(s, Ildst_imm_STRX, data_r, h.base,
1842	h.index_ext, h.index);
1843	break;
1844	default:
1845	g_assert_not_reached();
1846	}
1847	}
1848
1849	static void tgen_qemu_ld(TCGContext *s, TCGType data_type, TCGReg data_reg,
1850	TCGReg addr_reg, MemOpIdx oi)
1851	{
1852	TCGLabelQemuLdst *ldst;
1853	HostAddress h;
1854
1855	ldst = prepare_host_addr(s, &h, addr_reg, oi, true);
1856	tcg_out_qemu_ld_direct(s, get_memop(oi), data_type, data_reg, h);
1857
1858	if (ldst) {
1859	ldst->type = data_type;
1860	ldst->datalo_reg = data_reg;
1861	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1862	}
1863	}
1864
1865	static const TCGOutOpQemuLdSt outop_qemu_ld = {
1866	.base.static_constraint = C_O1_I1(r, r),
1867	.out = tgen_qemu_ld,
1868	};
1869
1870	static void tgen_qemu_st(TCGContext *s, TCGType data_type, TCGReg data_reg,
1871	TCGReg addr_reg, MemOpIdx oi)
1872	{
1873	TCGLabelQemuLdst *ldst;
1874	HostAddress h;
1875
1876	ldst = prepare_host_addr(s, &h, addr_reg, oi, false);
1877	tcg_out_qemu_st_direct(s, get_memop(oi), data_reg, h);
1878
1879	if (ldst) {
1880	ldst->type = data_type;
1881	ldst->datalo_reg = data_reg;
1882	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1883	}
1884	}
1885
1886	static const TCGOutOpQemuLdSt outop_qemu_st = {
1887	.base.static_constraint = C_O0_I2(rz, r),
1888	.out = tgen_qemu_st,
1889	};
1890
1891	static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi,
1892	TCGReg addr_reg, MemOpIdx oi, bool is_ld)
1893	{
1894	TCGLabelQemuLdst *ldst;
1895	HostAddress h;
1896	TCGReg base;
1897	bool use_pair;
1898
1899	ldst = prepare_host_addr(s, &h, addr_reg, oi, is_ld);
1900
1901	/* Compose the final address, as LDP/STP have no indexing. */
1902	if (h.index == TCG_REG_XZR) {
1903	base = h.base;
1904	} else {
1905	base = TCG_REG_TMP2;
1906	if (h.index_ext == TCG_TYPE_I32) {
1907	/* add base, base, index, uxtw */
1908	tcg_out_insn(s, addsub_ext, ADD, TCG_TYPE_I64, base,
1909	h.base, h.index, MO_32, 0);
1910	} else {
1911	/* add base, base, index */
1912	tcg_out_insn(s, addsub_shift, ADD, 1, base, h.base, h.index);
1913	}
1914	}
1915
1916	use_pair = h.aa.atom < MO_128 \|\| have_lse2;
1917
1918	if (!use_pair) {
1919	TCGLabel *label = NULL;
1920	TCGReg ll, lh, sl, sh;
1921	/*
1922	* Check for 16-byte alignment, taking into consideration the
1923	* alignment that has already been checked.
1924	*/
1925	int a_mask = 16 - (1 << h.aa.align);
1926
1927	if (a_mask > 0) {
1928	label = gen_new_label();
1929	tgen_brcondi(s, TCG_TYPE_I32, TCG_COND_TSTNE,
1930	addr_reg, a_mask, label);
1931	use_pair = true;
1932	}
1933
1934	if (is_ld) {
1935	/*
1936	* 16-byte atomicity without LSE2 requires LDXP+STXP loop:
1937	* ldxp lo, hi, [base]
1938	* stxp t0, lo, hi, [base]
1939	* cbnz t0, .-8
1940	* Require no overlap between data{lo,hi} and base.
1941	*/
1942	if (datalo == base \|\| datahi == base) {
1943	tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_TMP2, base);
1944	base = TCG_REG_TMP2;
1945	}
1946	ll = sl = datalo;
1947	lh = sh = datahi;
1948	} else {
1949	/*
1950	* 16-byte atomicity without LSE2 requires LDXP+STXP loop:
1951	* 1: ldxp t0, t1, [base]
1952	* stxp t0, lo, hi, [base]
1953	* cbnz t0, 1b
1954	*/
1955	tcg_debug_assert(base != TCG_REG_TMP0 && base != TCG_REG_TMP1);
1956	ll = TCG_REG_TMP0;
1957	lh = TCG_REG_TMP1;
1958	sl = datalo;
1959	sh = datahi;
1960	}
1961
1962	tcg_out_insn(s, stxp, LDXP, TCG_REG_XZR, ll, lh, base);
1963	tcg_out_insn(s, stxp, STXP, TCG_REG_TMP0, sl, sh, base);
1964	tcg_out_insn(s, cbz, CBNZ, 0, TCG_REG_TMP0, -2);
1965
1966	if (use_pair) {
1967	/* "b .+8", branching across the one insn of use_pair. */
1968	tcg_out_insn(s, branch, B, 2);
1969	tcg_out_label(s, label);
1970	}
1971	}
1972
1973	if (use_pair) {
1974	if (is_ld) {
1975	tcg_out_insn(s, ldstpair, LDP, datalo, datahi, base, 0, 1, 0);
1976	} else {
1977	tcg_out_insn(s, ldstpair, STP, datalo, datahi, base, 0, 1, 0);
1978	}
1979	}
1980
1981	if (ldst) {
1982	ldst->type = TCG_TYPE_I128;
1983	ldst->datalo_reg = datalo;
1984	ldst->datahi_reg = datahi;
1985	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1986	}
1987	}
1988
1989	static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo,
1990	TCGReg datahi, TCGReg addr_reg, MemOpIdx oi)
1991	{
1992	tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, true);
1993	}
1994
1995	static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = {
1996	.base.static_constraint = C_O2_I1(r, r, r),
1997	.out = tgen_qemu_ld2,
1998	};
1999
2000	static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo,
2001	TCGReg datahi, TCGReg addr_reg, MemOpIdx oi)
2002	{
2003	tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, false);
2004	}
2005
2006	static const TCGOutOpQemuLdSt2 outop_qemu_st2 = {
2007	.base.static_constraint = C_O0_I3(rz, rz, r),
2008	.out = tgen_qemu_st2,
2009	};
2010
2011	static const tcg_insn_unit *tb_ret_addr;
2012
2013	static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
2014	{
2015	const tcg_insn_unit *target;
2016	ptrdiff_t offset;
2017
2018	/* Reuse the zeroing that exists for goto_ptr. */
2019	if (a0 == 0) {
2020	target = tcg_code_gen_epilogue;
2021	} else {
2022	tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
2023	target = tb_ret_addr;
2024	}
2025
2026	offset = tcg_pcrel_diff(s, target) >> 2;
2027	if (offset == sextract64(offset, 0, 26)) {
2028	tcg_out_insn(s, branch, B, offset);
2029	} else {
2030	/*
2031	* Only x16/x17 generate BTI type Jump (2),
2032	* other registers generate BTI type Jump\|Call (3).
2033	*/
2034	QEMU_BUILD_BUG_ON(TCG_REG_TMP0 != TCG_REG_X16);
2035	tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP0, (intptr_t)target);
2036	tcg_out_insn(s, bcond_reg, BR, TCG_REG_TMP0);
2037	}
2038	}
2039
2040	static void tcg_out_goto_tb(TCGContext *s, int which)
2041	{
2042	/*
2043	* Direct branch, or indirect address load, will be patched
2044	* by tb_target_set_jmp_target. Assert indirect load offset
2045	* in range early, regardless of direct branch distance.
2046	*/
2047	intptr_t i_off = tcg_pcrel_diff(s, (void *)get_jmp_target_addr(s, which));
2048	tcg_debug_assert(i_off == sextract64(i_off, 0, 21));
2049
2050	set_jmp_insn_offset(s, which);
2051	tcg_out32(s, Ibranch_B);
2052	tcg_out_insn(s, bcond_reg, BR, TCG_REG_TMP0);
2053	set_jmp_reset_offset(s, which);
2054	tcg_out_bti(s, BTI_J);
2055	}
2056
2057	static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0)
2058	{
2059	tcg_out_insn(s, bcond_reg, BR, a0);
2060	}
2061
2062	void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
2063	uintptr_t jmp_rx, uintptr_t jmp_rw)
2064	{
2065	uintptr_t d_addr = tb->jmp_target_addr[n];
2066	ptrdiff_t d_offset = d_addr - jmp_rx;
2067	tcg_insn_unit insn;
2068
2069	/* Either directly branch, or indirect branch load. */
2070	if (d_offset == sextract64(d_offset, 0, 28)) {
2071	insn = deposit32(Ibranch_B, 0, 26, d_offset >> 2);
2072	} else {
2073	uintptr_t i_addr = (uintptr_t)&tb->jmp_target_addr[n];
2074	ptrdiff_t i_offset = i_addr - jmp_rx;
2075
2076	/* Note that we asserted this in range in tcg_out_goto_tb. */
2077	insn = deposit32(Ildlit_LDR \| TCG_REG_TMP0, 5, 19, i_offset >> 2);
2078	}
2079	qatomic_set((uint32_t *)jmp_rw, insn);
2080	flush_idcache_range(jmp_rx, jmp_rw, 4);
2081	}
2082
2083
2084	static void tgen_add(TCGContext *s, TCGType type,
2085	TCGReg a0, TCGReg a1, TCGReg a2)
2086	{
2087	tcg_out_insn(s, addsub_shift, ADD, type, a0, a1, a2);
2088	}
2089
2090	static void tgen_addi(TCGContext *s, TCGType type,
2091	TCGReg a0, TCGReg a1, tcg_target_long a2)
2092	{
2093	if (a2 >= 0) {
2094	tcg_out_insn(s, addsub_imm, ADDI, type, a0, a1, a2);
2095	} else {
2096	tcg_out_insn(s, addsub_imm, SUBI, type, a0, a1, -a2);
2097	}
2098	}
2099
2100	static const TCGOutOpBinary outop_add = {
2101	.base.static_constraint = C_O1_I2(r, r, rA),
2102	.out_rrr = tgen_add,
2103	.out_rri = tgen_addi,
2104	};
2105
2106	static void tgen_addco(TCGContext *s, TCGType type,
2107	TCGReg a0, TCGReg a1, TCGReg a2)
2108	{
2109	tcg_out_insn(s, addsub_shift, ADDS, type, a0, a1, a2);
2110	}
2111
2112	static void tgen_addco_imm(TCGContext *s, TCGType type,
2113	TCGReg a0, TCGReg a1, tcg_target_long a2)
2114	{
2115	if (a2 >= 0) {
2116	tcg_out_insn(s, addsub_imm, ADDSI, type, a0, a1, a2);
2117	} else {
2118	tcg_out_insn(s, addsub_imm, SUBSI, type, a0, a1, -a2);
2119	}
2120	}
2121
2122	static const TCGOutOpBinary outop_addco = {
2123	.base.static_constraint = C_O1_I2(r, r, rA),
2124	.out_rrr = tgen_addco,
2125	.out_rri = tgen_addco_imm,
2126	};
2127
2128	static void tgen_addci_rrr(TCGContext *s, TCGType type,
2129	TCGReg a0, TCGReg a1, TCGReg a2)
2130	{
2131	tcg_out_insn(s, rrr_sf, ADC, type, a0, a1, a2);
2132	}
2133
2134	static void tgen_addci_rri(TCGContext *s, TCGType type,
2135	TCGReg a0, TCGReg a1, tcg_target_long a2)
2136	{
2137	/*
2138	* Note that the only two constants we support are 0 and -1, and
2139	* that SBC = rn + ~rm + c, so adc -1 is sbc 0, and vice-versa.
2140	*/
2141	if (a2) {
2142	tcg_out_insn(s, rrr_sf, SBC, type, a0, a1, TCG_REG_XZR);
2143	} else {
2144	tcg_out_insn(s, rrr_sf, ADC, type, a0, a1, TCG_REG_XZR);
2145	}
2146	}
2147
2148	static const TCGOutOpAddSubCarry outop_addci = {
2149	.base.static_constraint = C_O1_I2(r, rz, rMZ),
2150	.out_rrr = tgen_addci_rrr,
2151	.out_rri = tgen_addci_rri,
2152	};
2153
2154	static void tgen_addcio(TCGContext *s, TCGType type,
2155	TCGReg a0, TCGReg a1, TCGReg a2)
2156	{
2157	tcg_out_insn(s, rrr_sf, ADCS, type, a0, a1, a2);
2158	}
2159
2160	static void tgen_addcio_imm(TCGContext *s, TCGType type,
2161	TCGReg a0, TCGReg a1, tcg_target_long a2)
2162	{
2163	/* Use SBCS w/0 for ADCS w/-1 -- see above. */
2164	if (a2) {
2165	tcg_out_insn(s, rrr_sf, SBCS, type, a0, a1, TCG_REG_XZR);
2166	} else {
2167	tcg_out_insn(s, rrr_sf, ADCS, type, a0, a1, TCG_REG_XZR);
2168	}
2169	}
2170
2171	static const TCGOutOpBinary outop_addcio = {
2172	.base.static_constraint = C_O1_I2(r, rz, rMZ),
2173	.out_rrr = tgen_addcio,
2174	.out_rri = tgen_addcio_imm,
2175	};
2176
2177	static void tcg_out_set_carry(TCGContext *s)
2178	{
2179	tcg_out_insn(s, addsub_shift, SUBS, TCG_TYPE_I32,
2180	TCG_REG_XZR, TCG_REG_XZR, TCG_REG_XZR);
2181	}
2182
2183	static void tgen_and(TCGContext *s, TCGType type,
2184	TCGReg a0, TCGReg a1, TCGReg a2)
2185	{
2186	tcg_out_insn(s, logic_shift, AND, type, a0, a1, a2);
2187	}
2188
2189	static void tgen_andi(TCGContext *s, TCGType type,
2190	TCGReg a0, TCGReg a1, tcg_target_long a2)
2191	{
2192	tcg_out_logicali(s, Ilogic_imm_ANDI, type, a0, a1, a2);
2193	}
2194
2195	static const TCGOutOpBinary outop_and = {
2196	.base.static_constraint = C_O1_I2(r, r, rL),
2197	.out_rrr = tgen_and,
2198	.out_rri = tgen_andi,
2199	};
2200
2201	static void tgen_andc(TCGContext *s, TCGType type,
2202	TCGReg a0, TCGReg a1, TCGReg a2)
2203	{
2204	tcg_out_insn(s, logic_shift, BIC, type, a0, a1, a2);
2205	}
2206
2207	static const TCGOutOpBinary outop_andc = {
2208	.base.static_constraint = C_O1_I2(r, r, r),
2209	.out_rrr = tgen_andc,
2210	};
2211
2212	static void tgen_clzctz(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2213	TCGReg a2, AArch64Insn insn)
2214	{
2215	tcg_out_cmp(s, type, TCG_COND_NE, a1, 0, true);
2216	tcg_out_insn_rr_sf(s, insn, type, TCG_REG_TMP0, a1);
2217	tcg_out_insn(s, csel, CSEL, type, a0, TCG_REG_TMP0, a2, TCG_COND_NE);
2218	}
2219
2220	static void tgen_clz(TCGContext *s, TCGType type,
2221	TCGReg a0, TCGReg a1, TCGReg a2)
2222	{
2223	tgen_clzctz(s, type, a0, a1, a2, Irr_sf_CLZ);
2224	}
2225
2226	static void tgen_clzctzi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2227	tcg_target_long a2, AArch64Insn insn)
2228	{
2229	if (a2 == (type == TCG_TYPE_I32 ? 32 : 64)) {
2230	tcg_out_insn_rr_sf(s, insn, type, a0, a1);
2231	return;
2232	}
2233
2234	tcg_out_cmp(s, type, TCG_COND_NE, a1, 0, true);
2235	tcg_out_insn_rr_sf(s, insn, type, a0, a1);
2236
2237	switch (a2) {
2238	case -1:
2239	tcg_out_insn(s, csel, CSINV, type, a0, a0, TCG_REG_XZR, TCG_COND_NE);
2240	break;
2241	case 0:
2242	tcg_out_insn(s, csel, CSEL, type, a0, a0, TCG_REG_XZR, TCG_COND_NE);
2243	break;
2244	default:
2245	tcg_out_movi(s, type, TCG_REG_TMP0, a2);
2246	tcg_out_insn(s, csel, CSEL, type, a0, a0, TCG_REG_TMP0, TCG_COND_NE);
2247	break;
2248	}
2249	}
2250
2251	static void tgen_clzi(TCGContext *s, TCGType type,
2252	TCGReg a0, TCGReg a1, tcg_target_long a2)
2253	{
2254	tgen_clzctzi(s, type, a0, a1, a2, Irr_sf_CLZ);
2255	}
2256
2257	static const TCGOutOpBinary outop_clz = {
2258	.base.static_constraint = C_O1_I2(r, r, rAL),
2259	.out_rrr = tgen_clz,
2260	.out_rri = tgen_clzi,
2261	};
2262
2263	static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags)
2264	{
2265	return cpuinfo & CPUINFO_CSSC ? C_O1_I1(r, r) : C_NotImplemented;
2266	}
2267
2268	static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2269	{
2270	tcg_out_insn(s, rr_sf, CNT, type, a0, a1);
2271	}
2272
2273	static const TCGOutOpUnary outop_ctpop = {
2274	.base.static_constraint = C_Dynamic,
2275	.base.dynamic_constraint = cset_ctpop,
2276	.out_rr = tgen_ctpop,
2277	};
2278
2279	static void tgen_ctz(TCGContext *s, TCGType type,
2280	TCGReg a0, TCGReg a1, TCGReg a2)
2281	{
2282	if (cpuinfo & CPUINFO_CSSC) {
2283	tgen_clzctz(s, type, a0, a1, a2, Irr_sf_CTZ);
2284	} else {
2285	tcg_out_insn(s, rr_sf, RBIT, type, TCG_REG_TMP0, a1);
2286	tgen_clzctz(s, type, a0, TCG_REG_TMP0, a2, Irr_sf_CLZ);
2287	}
2288	}
2289
2290	static void tgen_ctzi(TCGContext *s, TCGType type,
2291	TCGReg a0, TCGReg a1, tcg_target_long a2)
2292	{
2293	if (cpuinfo & CPUINFO_CSSC) {
2294	tgen_clzctzi(s, type, a0, a1, a2, Irr_sf_CTZ);
2295	} else {
2296	tcg_out_insn(s, rr_sf, RBIT, type, TCG_REG_TMP0, a1);
2297	tgen_clzctzi(s, type, a0, TCG_REG_TMP0, a2, Irr_sf_CLZ);
2298	}
2299	}
2300
2301	static const TCGOutOpBinary outop_ctz = {
2302	.base.static_constraint = C_O1_I2(r, r, rAL),
2303	.out_rrr = tgen_ctz,
2304	.out_rri = tgen_ctzi,
2305	};
2306
2307	static void tgen_divs(TCGContext *s, TCGType type,
2308	TCGReg a0, TCGReg a1, TCGReg a2)
2309	{
2310	tcg_out_insn(s, rrr, SDIV, type, a0, a1, a2);
2311	}
2312
2313	static const TCGOutOpBinary outop_divs = {
2314	.base.static_constraint = C_O1_I2(r, r, r),
2315	.out_rrr = tgen_divs,
2316	};
2317
2318	static const TCGOutOpDivRem outop_divs2 = {
2319	.base.static_constraint = C_NotImplemented,
2320	};
2321
2322	static void tgen_divu(TCGContext *s, TCGType type,
2323	TCGReg a0, TCGReg a1, TCGReg a2)
2324	{
2325	tcg_out_insn(s, rrr, UDIV, type, a0, a1, a2);
2326	}
2327
2328	static const TCGOutOpBinary outop_divu = {
2329	.base.static_constraint = C_O1_I2(r, r, r),
2330	.out_rrr = tgen_divu,
2331	};
2332
2333	static const TCGOutOpDivRem outop_divu2 = {
2334	.base.static_constraint = C_NotImplemented,
2335	};
2336
2337	static void tgen_eqv(TCGContext *s, TCGType type,
2338	TCGReg a0, TCGReg a1, TCGReg a2)
2339	{
2340	tcg_out_insn(s, logic_shift, EON, type, a0, a1, a2);
2341	}
2342
2343	static const TCGOutOpBinary outop_eqv = {
2344	.base.static_constraint = C_O1_I2(r, r, r),
2345	.out_rrr = tgen_eqv,
2346	};
2347
2348	static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
2349	{
2350	tcg_out_ubfm(s, TCG_TYPE_I64, a0, a1, 32, 63);
2351	}
2352
2353	static const TCGOutOpUnary outop_extrh_i64_i32 = {
2354	.base.static_constraint = C_O1_I1(r, r),
2355	.out_rr = tgen_extrh_i64_i32,
2356	};
2357
2358	static void tgen_mul(TCGContext *s, TCGType type,
2359	TCGReg a0, TCGReg a1, TCGReg a2)
2360	{
2361	tcg_out_insn(s, rrrr, MADD, type, a0, a1, a2, TCG_REG_XZR);
2362	}
2363
2364	static const TCGOutOpBinary outop_mul = {
2365	.base.static_constraint = C_O1_I2(r, r, r),
2366	.out_rrr = tgen_mul,
2367	};
2368
2369	static const TCGOutOpMul2 outop_muls2 = {
2370	.base.static_constraint = C_NotImplemented,
2371	};
2372
2373	static TCGConstraintSetIndex cset_mulh(TCGType type, unsigned flags)
2374	{
2375	return type == TCG_TYPE_I64 ? C_O1_I2(r, r, r) : C_NotImplemented;
2376	}
2377
2378	static void tgen_mulsh(TCGContext *s, TCGType type,
2379	TCGReg a0, TCGReg a1, TCGReg a2)
2380	{
2381	tcg_out_insn(s, rrr, SMULH, TCG_TYPE_I64, a0, a1, a2);
2382	}
2383
2384	static const TCGOutOpBinary outop_mulsh = {
2385	.base.static_constraint = C_Dynamic,
2386	.base.dynamic_constraint = cset_mulh,
2387	.out_rrr = tgen_mulsh,
2388	};
2389
2390	static const TCGOutOpMul2 outop_mulu2 = {
2391	.base.static_constraint = C_NotImplemented,
2392	};
2393
2394	static void tgen_muluh(TCGContext *s, TCGType type,
2395	TCGReg a0, TCGReg a1, TCGReg a2)
2396	{
2397	tcg_out_insn(s, rrr, UMULH, TCG_TYPE_I64, a0, a1, a2);
2398	}
2399
2400	static const TCGOutOpBinary outop_muluh = {
2401	.base.static_constraint = C_Dynamic,
2402	.base.dynamic_constraint = cset_mulh,
2403	.out_rrr = tgen_muluh,
2404	};
2405
2406	static const TCGOutOpBinary outop_nand = {
2407	.base.static_constraint = C_NotImplemented,
2408	};
2409
2410	static const TCGOutOpBinary outop_nor = {
2411	.base.static_constraint = C_NotImplemented,
2412	};
2413
2414	static void tgen_or(TCGContext *s, TCGType type,
2415	TCGReg a0, TCGReg a1, TCGReg a2)
2416	{
2417	tcg_out_insn(s, logic_shift, ORR, type, a0, a1, a2);
2418	}
2419
2420	static void tgen_ori(TCGContext *s, TCGType type,
2421	TCGReg a0, TCGReg a1, tcg_target_long a2)
2422	{
2423	tcg_out_logicali(s, Ilogic_imm_ORRI, type, a0, a1, a2);
2424	}
2425
2426	static const TCGOutOpBinary outop_or = {
2427	.base.static_constraint = C_O1_I2(r, r, rL),
2428	.out_rrr = tgen_or,
2429	.out_rri = tgen_ori,
2430	};
2431
2432	static void tgen_orc(TCGContext *s, TCGType type,
2433	TCGReg a0, TCGReg a1, TCGReg a2)
2434	{
2435	tcg_out_insn(s, logic_shift, ORN, type, a0, a1, a2);
2436	}
2437
2438	static const TCGOutOpBinary outop_orc = {
2439	.base.static_constraint = C_O1_I2(r, r, r),
2440	.out_rrr = tgen_orc,
2441	};
2442
2443	static void tgen_rems(TCGContext *s, TCGType type,
2444	TCGReg a0, TCGReg a1, TCGReg a2)
2445	{
2446	tcg_out_insn(s, rrr, SDIV, type, TCG_REG_TMP0, a1, a2);
2447	tcg_out_insn(s, rrrr, MSUB, type, a0, TCG_REG_TMP0, a2, a1);
2448	}
2449
2450	static const TCGOutOpBinary outop_rems = {
2451	.base.static_constraint = C_O1_I2(r, r, r),
2452	.out_rrr = tgen_rems,
2453	};
2454
2455	static void tgen_remu(TCGContext *s, TCGType type,
2456	TCGReg a0, TCGReg a1, TCGReg a2)
2457	{
2458	tcg_out_insn(s, rrr, UDIV, type, TCG_REG_TMP0, a1, a2);
2459	tcg_out_insn(s, rrrr, MSUB, type, a0, TCG_REG_TMP0, a2, a1);
2460	}
2461
2462	static const TCGOutOpBinary outop_remu = {
2463	.base.static_constraint = C_O1_I2(r, r, r),
2464	.out_rrr = tgen_remu,
2465	};
2466
2467	static const TCGOutOpBinary outop_rotl = {
2468	.base.static_constraint = C_NotImplemented,
2469	};
2470
2471	static void tgen_rotr(TCGContext *s, TCGType type,
2472	TCGReg a0, TCGReg a1, TCGReg a2)
2473	{
2474	tcg_out_insn(s, rrr, RORV, type, a0, a1, a2);
2475	}
2476
2477	static void tgen_rotri(TCGContext *s, TCGType type,
2478	TCGReg a0, TCGReg a1, tcg_target_long a2)
2479	{
2480	int max = type == TCG_TYPE_I32 ? 31 : 63;
2481	tcg_out_extr(s, type, a0, a1, a1, a2 & max);
2482	}
2483
2484	static const TCGOutOpBinary outop_rotr = {
2485	.base.static_constraint = C_O1_I2(r, r, ri),
2486	.out_rrr = tgen_rotr,
2487	.out_rri = tgen_rotri,
2488	};
2489
2490	static void tgen_sar(TCGContext *s, TCGType type,
2491	TCGReg a0, TCGReg a1, TCGReg a2)
2492	{
2493	tcg_out_insn(s, rrr, ASRV, type, a0, a1, a2);
2494	}
2495
2496	static void tgen_sari(TCGContext *s, TCGType type,
2497	TCGReg a0, TCGReg a1, tcg_target_long a2)
2498	{
2499	int max = type == TCG_TYPE_I32 ? 31 : 63;
2500	tcg_out_sbfm(s, type, a0, a1, a2 & max, max);
2501	}
2502
2503	static const TCGOutOpBinary outop_sar = {
2504	.base.static_constraint = C_O1_I2(r, r, ri),
2505	.out_rrr = tgen_sar,
2506	.out_rri = tgen_sari,
2507	};
2508
2509	static void tgen_shl(TCGContext *s, TCGType type,
2510	TCGReg a0, TCGReg a1, TCGReg a2)
2511	{
2512	tcg_out_insn(s, rrr, LSLV, type, a0, a1, a2);
2513	}
2514
2515	static void tgen_shli(TCGContext *s, TCGType type,
2516	TCGReg a0, TCGReg a1, tcg_target_long a2)
2517	{
2518	int max = type == TCG_TYPE_I32 ? 31 : 63;
2519	tcg_out_ubfm(s, type, a0, a1, -a2 & max, ~a2 & max);
2520	}
2521
2522	static const TCGOutOpBinary outop_shl = {
2523	.base.static_constraint = C_O1_I2(r, r, ri),
2524	.out_rrr = tgen_shl,
2525	.out_rri = tgen_shli,
2526	};
2527
2528	static void tgen_shr(TCGContext *s, TCGType type,
2529	TCGReg a0, TCGReg a1, TCGReg a2)
2530	{
2531	tcg_out_insn(s, rrr, LSRV, type, a0, a1, a2);
2532	}
2533
2534	static void tgen_shri(TCGContext *s, TCGType type,
2535	TCGReg a0, TCGReg a1, tcg_target_long a2)
2536	{
2537	int max = type == TCG_TYPE_I32 ? 31 : 63;
2538	tcg_out_ubfm(s, type, a0, a1, a2 & max, max);
2539	}
2540
2541	static const TCGOutOpBinary outop_shr = {
2542	.base.static_constraint = C_O1_I2(r, r, ri),
2543	.out_rrr = tgen_shr,
2544	.out_rri = tgen_shri,
2545	};
2546
2547	static void tgen_sub(TCGContext *s, TCGType type,
2548	TCGReg a0, TCGReg a1, TCGReg a2)
2549	{
2550	tcg_out_insn(s, addsub_shift, SUB, type, a0, a1, a2);
2551	}
2552
2553	static const TCGOutOpSubtract outop_sub = {
2554	.base.static_constraint = C_O1_I2(r, r, r),
2555	.out_rrr = tgen_sub,
2556	};
2557
2558	static void tgen_subbo_rrr(TCGContext *s, TCGType type,
2559	TCGReg a0, TCGReg a1, TCGReg a2)
2560	{
2561	tcg_out_insn(s, addsub_shift, SUBS, type, a0, a1, a2);
2562	}
2563
2564	static void tgen_subbo_rri(TCGContext *s, TCGType type,
2565	TCGReg a0, TCGReg a1, tcg_target_long a2)
2566	{
2567	if (a2 >= 0) {
2568	tcg_out_insn(s, addsub_imm, SUBSI, type, a0, a1, a2);
2569	} else {
2570	tcg_out_insn(s, addsub_imm, ADDSI, type, a0, a1, -a2);
2571	}
2572	}
2573
2574	static void tgen_subbo_rir(TCGContext *s, TCGType type,
2575	TCGReg a0, tcg_target_long a1, TCGReg a2)
2576	{
2577	tgen_subbo_rrr(s, type, a0, TCG_REG_XZR, a2);
2578	}
2579
2580	static void tgen_subbo_rii(TCGContext *s, TCGType type,
2581	TCGReg a0, tcg_target_long a1, tcg_target_long a2)
2582	{
2583	if (a2 == 0) {
2584	tgen_subbo_rrr(s, type, a0, TCG_REG_XZR, TCG_REG_XZR);
2585	return;
2586	}
2587
2588	/*
2589	* We want to allow a1 to be zero for the benefit of negation via
2590	* subtraction. However, that leaves open the possibility of
2591	* adding 0 +/- const, and the immediate add/sub instructions
2592	* encode XSP not XZR. Since we have 0 - non-zero, borrow is
2593	* always set.
2594	*/
2595	tcg_out_movi(s, type, a0, -a2);
2596	tcg_out_set_borrow(s);
2597	}
2598
2599	static const TCGOutOpAddSubCarry outop_subbo = {
2600	.base.static_constraint = C_O1_I2(r, rZ, rA),
2601	.out_rrr = tgen_subbo_rrr,
2602	.out_rri = tgen_subbo_rri,
2603	.out_rir = tgen_subbo_rir,
2604	.out_rii = tgen_subbo_rii,
2605	};
2606
2607	static void tgen_subbi_rrr(TCGContext *s, TCGType type,
2608	TCGReg a0, TCGReg a1, TCGReg a2)
2609	{
2610	tcg_out_insn(s, rrr_sf, SBC, type, a0, a1, a2);
2611	}
2612
2613	static void tgen_subbi_rri(TCGContext *s, TCGType type,
2614	TCGReg a0, TCGReg a1, tcg_target_long a2)
2615	{
2616	tgen_addci_rri(s, type, a0, a1, ~a2);
2617	}
2618
2619	static const TCGOutOpAddSubCarry outop_subbi = {
2620	.base.static_constraint = C_O1_I2(r, rz, rMZ),
2621	.out_rrr = tgen_subbi_rrr,
2622	.out_rri = tgen_subbi_rri,
2623	};
2624
2625	static void tgen_subbio_rrr(TCGContext *s, TCGType type,
2626	TCGReg a0, TCGReg a1, TCGReg a2)
2627	{
2628	tcg_out_insn(s, rrr_sf, SBCS, type, a0, a1, a2);
2629	}
2630
2631	static void tgen_subbio_rri(TCGContext *s, TCGType type,
2632	TCGReg a0, TCGReg a1, tcg_target_long a2)
2633	{
2634	tgen_addcio_imm(s, type, a0, a1, ~a2);
2635	}
2636
2637	static const TCGOutOpAddSubCarry outop_subbio = {
2638	.base.static_constraint = C_O1_I2(r, rz, rMZ),
2639	.out_rrr = tgen_subbio_rrr,
2640	.out_rri = tgen_subbio_rri,
2641	};
2642
2643	static void tcg_out_set_borrow(TCGContext *s)
2644	{
2645	tcg_out_insn(s, addsub_shift, ADDS, TCG_TYPE_I32,
2646	TCG_REG_XZR, TCG_REG_XZR, TCG_REG_XZR);
2647	}
2648
2649	static TCGConstraintSetIndex cset_sminmax(TCGType type, unsigned flags)
2650	{
2651	return cpuinfo & CPUINFO_CSSC ? C_O1_I2(r, r, rS) : C_NotImplemented;
2652	}
2653
2654	static void tgen_smax(TCGContext *s, TCGType type,
2655	TCGReg a0, TCGReg a1, TCGReg a2)
2656	{
2657	tcg_out_insn(s, rrr, SMAX, type, a0, a1, a2);
2658	}
2659
2660	static void tgen_smaxi(TCGContext *s, TCGType type,
2661	TCGReg a0, TCGReg a1, tcg_target_long a2)
2662	{
2663	tcg_out_insn(s, minmax_imm, SMAXI, type, a0, a1, a2);
2664	}
2665
2666	static const TCGOutOpBinary outop_smax = {
2667	.base.static_constraint = C_Dynamic,
2668	.base.dynamic_constraint = cset_sminmax,
2669	.out_rrr = tgen_smax,
2670	.out_rri = tgen_smaxi,
2671	};
2672
2673	static void tgen_smin(TCGContext *s, TCGType type,
2674	TCGReg a0, TCGReg a1, TCGReg a2)
2675	{
2676	tcg_out_insn(s, rrr, SMIN, type, a0, a1, a2);
2677	}
2678
2679	static void tgen_smini(TCGContext *s, TCGType type,
2680	TCGReg a0, TCGReg a1, tcg_target_long a2)
2681	{
2682	tcg_out_insn(s, minmax_imm, SMINI, type, a0, a1, a2);
2683	}
2684
2685	static const TCGOutOpBinary outop_smin = {
2686	.base.static_constraint = C_Dynamic,
2687	.base.dynamic_constraint = cset_sminmax,
2688	.out_rrr = tgen_smin,
2689	.out_rri = tgen_smini,
2690	};
2691
2692	static TCGConstraintSetIndex cset_uminmax(TCGType type, unsigned flags)
2693	{
2694	return cpuinfo & CPUINFO_CSSC ? C_O1_I2(r, r, rU) : C_NotImplemented;
2695	}
2696
2697	static void tgen_umax(TCGContext *s, TCGType type,
2698	TCGReg a0, TCGReg a1, TCGReg a2)
2699	{
2700	tcg_out_insn(s, rrr, UMAX, type, a0, a1, a2);
2701	}
2702
2703	static void tgen_umaxi(TCGContext *s, TCGType type,
2704	TCGReg a0, TCGReg a1, tcg_target_long a2)
2705	{
2706	tcg_out_insn(s, minmax_imm, UMAXI, type, a0, a1, a2);
2707	}
2708
2709	static const TCGOutOpBinary outop_umax = {
2710	.base.static_constraint = C_Dynamic,
2711	.base.dynamic_constraint = cset_uminmax,
2712	.out_rrr = tgen_umax,
2713	.out_rri = tgen_umaxi,
2714	};
2715
2716	static void tgen_umin(TCGContext *s, TCGType type,
2717	TCGReg a0, TCGReg a1, TCGReg a2)
2718	{
2719	tcg_out_insn(s, rrr, UMIN, type, a0, a1, a2);
2720	}
2721
2722	static void tgen_umini(TCGContext *s, TCGType type,
2723	TCGReg a0, TCGReg a1, tcg_target_long a2)
2724	{
2725	tcg_out_insn(s, minmax_imm, UMINI, type, a0, a1, a2);
2726	}
2727
2728	static const TCGOutOpBinary outop_umin = {
2729	.base.static_constraint = C_Dynamic,
2730	.base.dynamic_constraint = cset_uminmax,
2731	.out_rrr = tgen_umin,
2732	.out_rri = tgen_umini,
2733	};
2734
2735	static void tgen_xor(TCGContext *s, TCGType type,
2736	TCGReg a0, TCGReg a1, TCGReg a2)
2737	{
2738	tcg_out_insn(s, logic_shift, EOR, type, a0, a1, a2);
2739	}
2740
2741	static void tgen_xori(TCGContext *s, TCGType type,
2742	TCGReg a0, TCGReg a1, tcg_target_long a2)
2743	{
2744	tcg_out_logicali(s, Ilogic_imm_EORI, type, a0, a1, a2);
2745	}
2746
2747	static const TCGOutOpBinary outop_xor = {
2748	.base.static_constraint = C_O1_I2(r, r, rL),
2749	.out_rrr = tgen_xor,
2750	.out_rri = tgen_xori,
2751	};
2752
2753	static void tgen_bswap16(TCGContext *s, TCGType type,
2754	TCGReg a0, TCGReg a1, unsigned flags)
2755	{
2756	tcg_out_rev(s, TCG_TYPE_I32, MO_16, a0, a1);
2757	if (flags & TCG_BSWAP_OS) {
2758	/* Output must be sign-extended. */
2759	tcg_out_ext16s(s, type, a0, a0);
2760	} else if ((flags & (TCG_BSWAP_IZ \| TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
2761	/* Output must be zero-extended, but input isn't. */
2762	tcg_out_ext16u(s, a0, a0);
2763	}
2764	}
2765
2766	static const TCGOutOpBswap outop_bswap16 = {
2767	.base.static_constraint = C_O1_I1(r, r),
2768	.out_rr = tgen_bswap16,
2769	};
2770
2771	static void tgen_bswap32(TCGContext *s, TCGType type,
2772	TCGReg a0, TCGReg a1, unsigned flags)
2773	{
2774	tcg_out_rev(s, TCG_TYPE_I32, MO_32, a0, a1);
2775	if (flags & TCG_BSWAP_OS) {
2776	tcg_out_ext32s(s, a0, a0);
2777	}
2778	}
2779
2780	static const TCGOutOpBswap outop_bswap32 = {
2781	.base.static_constraint = C_O1_I1(r, r),
2782	.out_rr = tgen_bswap32,
2783	};
2784
2785	static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2786	{
2787	tcg_out_rev(s, TCG_TYPE_I64, MO_64, a0, a1);
2788	}
2789
2790	static const TCGOutOpUnary outop_bswap64 = {
2791	.base.static_constraint = C_O1_I1(r, r),
2792	.out_rr = tgen_bswap64,
2793	};
2794
2795	static const TCGOutOpUnary outop_revbit8 = {
2796	.base.static_constraint = C_NotImplemented,
2797	};
2798
2799	static void tgen_revbit32(TCGContext *s, TCGType type,
2800	TCGReg a0, TCGReg a1, unsigned flags)
2801	{
2802	tcg_out_insn(s, rr_sf, RBIT, TCG_TYPE_I32, a0, a1);
2803	if (flags & TCG_BSWAP_OS) {
2804	tcg_out_ext32s(s, a0, a0);
2805	}
2806	}
2807
2808	static const TCGOutOpBswap outop_revbit32 = {
2809	.base.static_constraint = C_O1_I1(r, r),
2810	.out_rr = tgen_revbit32,
2811	};
2812
2813	static void tgen_revbit64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2814	{
2815	tcg_out_insn(s, rr_sf, RBIT, TCG_TYPE_I64, a0, a1);
2816	}
2817
2818	static const TCGOutOpUnary outop_revbit64 = {
2819	.base.static_constraint = C_O1_I1(r, r),
2820	.out_rr = tgen_revbit64,
2821	};
2822
2823	static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2824	{
2825	tgen_sub(s, type, a0, TCG_REG_XZR, a1);
2826	}
2827
2828	static const TCGOutOpUnary outop_neg = {
2829	.base.static_constraint = C_O1_I1(r, r),
2830	.out_rr = tgen_neg,
2831	};
2832
2833	static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2834	{
2835	tgen_orc(s, type, a0, TCG_REG_XZR, a1);
2836	}
2837
2838	static const TCGOutOpUnary outop_not = {
2839	.base.static_constraint = C_O1_I1(r, r),
2840	.out_rr = tgen_not,
2841	};
2842
2843	static void tgen_cset(TCGContext *s, TCGCond cond, TCGReg ret)
2844	{
2845	/* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond). */
2846	tcg_out_insn(s, csel, CSINC, TCG_TYPE_I32, ret, TCG_REG_XZR,
2847	TCG_REG_XZR, tcg_invert_cond(cond));
2848	}
2849
2850	static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
2851	TCGReg a0, TCGReg a1, TCGReg a2)
2852	{
2853	tgen_cmp(s, type, cond, a1, a2);
2854	tgen_cset(s, cond, a0);
2855	}
2856
2857	static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond,
2858	TCGReg a0, TCGReg a1, tcg_target_long a2)
2859	{
2860	tgen_cmpi(s, type, cond, a1, a2);
2861	tgen_cset(s, cond, a0);
2862	}
2863
2864	static const TCGOutOpSetcond outop_setcond = {
2865	.base.static_constraint = C_O1_I2(r, r, rC),
2866	.out_rrr = tgen_setcond,
2867	.out_rri = tgen_setcondi,
2868	};
2869
2870	static void tgen_csetm(TCGContext *s, TCGType ext, TCGCond cond, TCGReg ret)
2871	{
2872	/* Use CSETM alias of CSINV Wd, WZR, WZR, invert(cond). */
2873	tcg_out_insn(s, csel, CSINV, ext, ret, TCG_REG_XZR,
2874	TCG_REG_XZR, tcg_invert_cond(cond));
2875	}
2876
2877	static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond,
2878	TCGReg a0, TCGReg a1, TCGReg a2)
2879	{
2880	tgen_cmp(s, type, cond, a1, a2);
2881	tgen_csetm(s, type, cond, a0);
2882	}
2883
2884	static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond,
2885	TCGReg a0, TCGReg a1, tcg_target_long a2)
2886	{
2887	tgen_cmpi(s, type, cond, a1, a2);
2888	tgen_csetm(s, type, cond, a0);
2889	}
2890
2891	static const TCGOutOpSetcond outop_negsetcond = {
2892	.base.static_constraint = C_O1_I2(r, r, rC),
2893	.out_rrr = tgen_negsetcond,
2894	.out_rri = tgen_negsetcondi,
2895	};
2896
2897	static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond,
2898	TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2,
2899	TCGArg vt, bool const_vt, TCGArg vf, bool const_vf)
2900	{
2901	tcg_out_cmp(s, type, cond, c1, c2, const_c2);
2902	tcg_out_insn(s, csel, CSEL, type, ret, vt, vf, cond);
2903	}
2904
2905	static const TCGOutOpMovcond outop_movcond = {
2906	.base.static_constraint = C_O1_I4(r, r, rC, rz, rz),
2907	.out = tgen_movcond,
2908	};
2909
2910	static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2911	TCGReg a2, unsigned ofs, unsigned len)
2912	{
2913	unsigned mask = type == TCG_TYPE_I32 ? 31 : 63;
2914
2915	/*
2916	* Since we can't support "0Z" as a constraint, we allow a1 in
2917	* any register. Fix things up as if a matching constraint.
2918	*/
2919	if (a0 != a1) {
2920	if (a0 == a2) {
2921	tcg_out_mov(s, type, TCG_REG_TMP0, a2);
2922	a2 = TCG_REG_TMP0;
2923	}
2924	tcg_out_mov(s, type, a0, a1);
2925	}
2926	tcg_out_bfm(s, type, a0, a2, -ofs & mask, len - 1);
2927	}
2928
2929	static void tgen_depositi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2930	tcg_target_long a2, unsigned ofs, unsigned len)
2931	{
2932	tgen_andi(s, type, a0, a1, ~MAKE_64BIT_MASK(ofs, len));
2933	}
2934
2935	static void tgen_depositz(TCGContext *s, TCGType type, TCGReg a0, TCGReg a2,
2936	unsigned ofs, unsigned len)
2937	{
2938	int max = type == TCG_TYPE_I32 ? 31 : 63;
2939	tcg_out_ubfm(s, type, a0, a2, -ofs & max, len - 1);
2940	}
2941
2942	static const TCGOutOpDeposit outop_deposit = {
2943	.base.static_constraint = C_O1_I2(r, rZ, rZ),
2944	.out_rrr = tgen_deposit,
2945	.out_rri = tgen_depositi,
2946	.out_rzr = tgen_depositz,
2947	};
2948
2949	static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2950	unsigned ofs, unsigned len)
2951	{
2952	if (ofs == 0) {
2953	uint64_t mask = MAKE_64BIT_MASK(0, len);
2954	tcg_out_logicali(s, Ilogic_imm_ANDI, type, a0, a1, mask);
2955	} else {
2956	tcg_out_ubfm(s, type, a0, a1, ofs, ofs + len - 1);
2957	}
2958	}
2959
2960	static const TCGOutOpExtract outop_extract = {
2961	.base.static_constraint = C_O1_I1(r, r),
2962	.out_rr = tgen_extract,
2963	};
2964
2965	static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2966	unsigned ofs, unsigned len)
2967	{
2968	tcg_out_sbfm(s, type, a0, a1, ofs, ofs + len - 1);
2969	}
2970
2971	static const TCGOutOpExtract outop_sextract = {
2972	.base.static_constraint = C_O1_I1(r, r),
2973	.out_rr = tgen_sextract,
2974	};
2975
2976	static void tgen_extract2(TCGContext *s, TCGType type, TCGReg a0,
2977	TCGReg a1, TCGReg a2, unsigned shr)
2978	{
2979	tcg_out_extr(s, type, a0, a2, a1, shr);
2980	}
2981
2982	static const TCGOutOpExtract2 outop_extract2 = {
2983	.base.static_constraint = C_O1_I2(r, rz, rz),
2984	.out_rrr = tgen_extract2,
2985	};
2986
2987	static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest,
2988	TCGReg base, ptrdiff_t offset)
2989	{
2990	tcg_out_ldst(s, Ildst_imm_LDRB, dest, base, offset, 0);
2991	}
2992
2993	static const TCGOutOpLoad outop_ld8u = {
2994	.base.static_constraint = C_O1_I1(r, r),
2995	.out = tgen_ld8u,
2996	};
2997
2998	static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest,
2999	TCGReg base, ptrdiff_t offset)
3000	{
3001	AArch64Insn insn = type == TCG_TYPE_I32 ? Ildst_imm_LDRSBW
3002	: Ildst_imm_LDRSBX;
3003	tcg_out_ldst(s, insn, dest, base, offset, 0);
3004	}
3005
3006	static const TCGOutOpLoad outop_ld8s = {
3007	.base.static_constraint = C_O1_I1(r, r),
3008	.out = tgen_ld8s,
3009	};
3010
3011	static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest,
3012	TCGReg base, ptrdiff_t offset)
3013	{
3014	tcg_out_ldst(s, Ildst_imm_LDRH, dest, base, offset, 1);
3015	}
3016
3017	static const TCGOutOpLoad outop_ld16u = {
3018	.base.static_constraint = C_O1_I1(r, r),
3019	.out = tgen_ld16u,
3020	};
3021
3022	static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest,
3023	TCGReg base, ptrdiff_t offset)
3024	{
3025	AArch64Insn insn = type == TCG_TYPE_I32 ? Ildst_imm_LDRSHW
3026	: Ildst_imm_LDRSHX;
3027	tcg_out_ldst(s, insn, dest, base, offset, 1);
3028	}
3029
3030	static const TCGOutOpLoad outop_ld16s = {
3031	.base.static_constraint = C_O1_I1(r, r),
3032	.out = tgen_ld16s,
3033	};
3034
3035	static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest,
3036	TCGReg base, ptrdiff_t offset)
3037	{
3038	tcg_out_ldst(s, Ildst_imm_LDRW, dest, base, offset, 2);
3039	}
3040
3041	static const TCGOutOpLoad outop_ld32u = {
3042	.base.static_constraint = C_O1_I1(r, r),
3043	.out = tgen_ld32u,
3044	};
3045
3046	static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest,
3047	TCGReg base, ptrdiff_t offset)
3048	{
3049	tcg_out_ldst(s, Ildst_imm_LDRSWX, dest, base, offset, 2);
3050	}
3051
3052	static const TCGOutOpLoad outop_ld32s = {
3053	.base.static_constraint = C_O1_I1(r, r),
3054	.out = tgen_ld32s,
3055	};
3056
3057	static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data,
3058	TCGReg base, ptrdiff_t offset)
3059	{
3060	tcg_out_ldst(s, Ildst_imm_STRB, data, base, offset, 0);
3061	}
3062
3063	static const TCGOutOpStore outop_st8 = {
3064	.base.static_constraint = C_O0_I2(rz, r),
3065	.out_r = tgen_st8_r,
3066	};
3067
3068	static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data,
3069	TCGReg base, ptrdiff_t offset)
3070	{
3071	tcg_out_ldst(s, Ildst_imm_STRH, data, base, offset, 1);
3072	}
3073
3074	static const TCGOutOpStore outop_st16 = {
3075	.base.static_constraint = C_O0_I2(rz, r),
3076	.out_r = tgen_st16_r,
3077	};
3078
3079	static const TCGOutOpStore outop_st = {
3080	.base.static_constraint = C_O0_I2(rz, r),
3081	.out_r = tcg_out_st,
3082	};
3083
3084	static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
3085	unsigned vecl, unsigned vece,
3086	const TCGArg args[TCG_MAX_OP_ARGS],
3087	const int const_args[TCG_MAX_OP_ARGS])
3088	{
3089	static const AArch64Insn cmp_vec_insn[16] = {
3090	[TCG_COND_EQ] = Iqrrr_e_CMEQ,
3091	[TCG_COND_GT] = Iqrrr_e_CMGT,
3092	[TCG_COND_GE] = Iqrrr_e_CMGE,
3093	[TCG_COND_GTU] = Iqrrr_e_CMHI,
3094	[TCG_COND_GEU] = Iqrrr_e_CMHS,
3095	};
3096	static const AArch64Insn cmp_scalar_insn[16] = {
3097	[TCG_COND_EQ] = Irrr_e_CMEQ,
3098	[TCG_COND_GT] = Irrr_e_CMGT,
3099	[TCG_COND_GE] = Irrr_e_CMGE,
3100	[TCG_COND_GTU] = Irrr_e_CMHI,
3101	[TCG_COND_GEU] = Irrr_e_CMHS,
3102	};
3103	static const AArch64Insn cmp0_vec_insn[16] = {
3104	[TCG_COND_EQ] = Iqrr_e_CMEQ0,
3105	[TCG_COND_GT] = Iqrr_e_CMGT0,
3106	[TCG_COND_GE] = Iqrr_e_CMGE0,
3107	[TCG_COND_LT] = Iqrr_e_CMLT0,
3108	[TCG_COND_LE] = Iqrr_e_CMLE0,
3109	};
3110	static const AArch64Insn cmp0_scalar_insn[16] = {
3111	[TCG_COND_EQ] = Isimd_rr_CMEQ0,
3112	[TCG_COND_GT] = Isimd_rr_CMGT0,
3113	[TCG_COND_GE] = Isimd_rr_CMGE0,
3114	[TCG_COND_LT] = Isimd_rr_CMLT0,
3115	[TCG_COND_LE] = Isimd_rr_CMLE0,
3116	};
3117
3118	TCGType type = vecl + TCG_TYPE_V64;
3119	unsigned is_q = vecl;
3120	bool is_scalar = !is_q && vece == MO_64;
3121	TCGArg a0, a1, a2, a3;
3122	int cmode, imm8;
3123
3124	a0 = args[0];
3125	a1 = args[1];
3126	a2 = args[2];
3127
3128	switch (opc) {
3129	case INDEX_op_ld_vec:
3130	tcg_out_ld(s, type, a0, a1, a2);
3131	break;
3132	case INDEX_op_st_vec:
3133	tcg_out_st(s, type, a0, a1, a2);
3134	break;
3135	case INDEX_op_dupm_vec:
3136	tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
3137	break;
3138	case INDEX_op_add_vec:
3139	if (is_scalar) {
3140	tcg_out_insn(s, rrr_e, ADD, vece, a0, a1, a2);
3141	} else {
3142	tcg_out_insn(s, qrrr_e, ADD, is_q, vece, a0, a1, a2);
3143	}
3144	break;
3145	case INDEX_op_sub_vec:
3146	if (is_scalar) {
3147	tcg_out_insn(s, rrr_e, SUB, vece, a0, a1, a2);
3148	} else {
3149	tcg_out_insn(s, qrrr_e, SUB, is_q, vece, a0, a1, a2);
3150	}
3151	break;
3152	case INDEX_op_mul_vec:
3153	tcg_out_insn(s, qrrr_e, MUL, is_q, vece, a0, a1, a2);
3154	break;
3155	case INDEX_op_neg_vec:
3156	if (is_scalar) {
3157	tcg_out_insn(s, simd_rr, NEG, vece, a0, a1);
3158	} else {
3159	tcg_out_insn(s, qrr_e, NEG, is_q, vece, a0, a1);
3160	}
3161	break;
3162	case INDEX_op_abs_vec:
3163	if (is_scalar) {
3164	tcg_out_insn(s, simd_rr, ABS, vece, a0, a1);
3165	} else {
3166	tcg_out_insn(s, qrr_e, ABS, is_q, vece, a0, a1);
3167	}
3168	break;
3169	case INDEX_op_and_vec:
3170	if (const_args[2]) {
3171	is_shimm1632(~a2, &cmode, &imm8);
3172	if (a0 == a1) {
3173	tcg_out_insn(s, simd_imm, BIC, is_q, a0, 0, cmode, imm8);
3174	return;
3175	}
3176	tcg_out_insn(s, simd_imm, MVNI, is_q, a0, 0, cmode, imm8);
3177	a2 = a0;
3178	}
3179	tcg_out_insn(s, qrrr_e, AND, is_q, 0, a0, a1, a2);
3180	break;
3181	case INDEX_op_or_vec:
3182	if (const_args[2]) {
3183	is_shimm1632(a2, &cmode, &imm8);
3184	if (a0 == a1) {
3185	tcg_out_insn(s, simd_imm, ORR, is_q, a0, 0, cmode, imm8);
3186	return;
3187	}
3188	tcg_out_insn(s, simd_imm, MOVI, is_q, a0, 0, cmode, imm8);
3189	a2 = a0;
3190	}
3191	tcg_out_insn(s, qrrr_e, ORR, is_q, 0, a0, a1, a2);
3192	break;
3193	case INDEX_op_andc_vec:
3194	if (const_args[2]) {
3195	is_shimm1632(a2, &cmode, &imm8);
3196	if (a0 == a1) {
3197	tcg_out_insn(s, simd_imm, BIC, is_q, a0, 0, cmode, imm8);
3198	return;
3199	}
3200	tcg_out_insn(s, simd_imm, MOVI, is_q, a0, 0, cmode, imm8);
3201	a2 = a0;
3202	}
3203	tcg_out_insn(s, qrrr_e, BIC, is_q, 0, a0, a1, a2);
3204	break;
3205	case INDEX_op_orc_vec:
3206	if (const_args[2]) {
3207	is_shimm1632(~a2, &cmode, &imm8);
3208	if (a0 == a1) {
3209	tcg_out_insn(s, simd_imm, ORR, is_q, a0, 0, cmode, imm8);
3210	return;
3211	}
3212	tcg_out_insn(s, simd_imm, MVNI, is_q, a0, 0, cmode, imm8);
3213	a2 = a0;
3214	}
3215	tcg_out_insn(s, qrrr_e, ORN, is_q, 0, a0, a1, a2);
3216	break;
3217	case INDEX_op_xor_vec:
3218	tcg_out_insn(s, qrrr_e, EOR, is_q, 0, a0, a1, a2);
3219	break;
3220	case INDEX_op_ssadd_vec:
3221	if (is_scalar) {
3222	tcg_out_insn(s, rrr_e, SQADD, vece, a0, a1, a2);
3223	} else {
3224	tcg_out_insn(s, qrrr_e, SQADD, is_q, vece, a0, a1, a2);
3225	}
3226	break;
3227	case INDEX_op_sssub_vec:
3228	if (is_scalar) {
3229	tcg_out_insn(s, rrr_e, SQSUB, vece, a0, a1, a2);
3230	} else {
3231	tcg_out_insn(s, qrrr_e, SQSUB, is_q, vece, a0, a1, a2);
3232	}
3233	break;
3234	case INDEX_op_usadd_vec:
3235	if (is_scalar) {
3236	tcg_out_insn(s, rrr_e, UQADD, vece, a0, a1, a2);
3237	} else {
3238	tcg_out_insn(s, qrrr_e, UQADD, is_q, vece, a0, a1, a2);
3239	}
3240	break;
3241	case INDEX_op_ussub_vec:
3242	if (is_scalar) {
3243	tcg_out_insn(s, rrr_e, UQSUB, vece, a0, a1, a2);
3244	} else {
3245	tcg_out_insn(s, qrrr_e, UQSUB, is_q, vece, a0, a1, a2);
3246	}
3247	break;
3248	case INDEX_op_smax_vec:
3249	tcg_out_insn(s, qrrr_e, SMAX, is_q, vece, a0, a1, a2);
3250	break;
3251	case INDEX_op_smin_vec:
3252	tcg_out_insn(s, qrrr_e, SMIN, is_q, vece, a0, a1, a2);
3253	break;
3254	case INDEX_op_umax_vec:
3255	tcg_out_insn(s, qrrr_e, UMAX, is_q, vece, a0, a1, a2);
3256	break;
3257	case INDEX_op_umin_vec:
3258	tcg_out_insn(s, qrrr_e, UMIN, is_q, vece, a0, a1, a2);
3259	break;
3260	case INDEX_op_not_vec:
3261	tcg_out_insn(s, qrr_e, NOT, is_q, 0, a0, a1);
3262	break;
3263	case INDEX_op_shli_vec:
3264	if (is_scalar) {
3265	tcg_out_insn(s, q_shift, SHL, a0, a1, a2 + (8 << vece));
3266	} else {
3267	tcg_out_insn(s, simd_shift_imm, SHL, is_q, a0, a1,
3268	a2 + (8 << vece));
3269	}
3270	break;
3271	case INDEX_op_shri_vec:
3272	if (is_scalar) {
3273	tcg_out_insn(s, q_shift, USHR, a0, a1, (16 << vece) - a2);
3274	} else {
3275	tcg_out_insn(s, simd_shift_imm, USHR, is_q, a0, a1,
3276	(16 << vece) - a2);
3277	}
3278	break;
3279	case INDEX_op_sari_vec:
3280	if (is_scalar) {
3281	tcg_out_insn(s, q_shift, SSHR, a0, a1, (16 << vece) - a2);
3282	} else {
3283	tcg_out_insn(s, simd_shift_imm, SSHR, is_q, a0, a1,
3284	(16 << vece) - a2);
3285	}
3286	break;
3287	case INDEX_op_aa64_sli_vec:
3288	if (is_scalar) {
3289	tcg_out_insn(s, q_shift, SLI, a0, a2, args[3] + (8 << vece));
3290	} else {
3291	tcg_out_insn(s, simd_shift_imm, SLI, is_q, a0, a2,
3292	args[3] + (8 << vece));
3293	}
3294	break;
3295	case INDEX_op_shlv_vec:
3296	if (is_scalar) {
3297	tcg_out_insn(s, rrr_e, USHL, vece, a0, a1, a2);
3298	} else {
3299	tcg_out_insn(s, qrrr_e, USHL, is_q, vece, a0, a1, a2);
3300	}
3301	break;
3302	case INDEX_op_aa64_sshl_vec:
3303	if (is_scalar) {
3304	tcg_out_insn(s, rrr_e, SSHL, vece, a0, a1, a2);
3305	} else {
3306	tcg_out_insn(s, qrrr_e, SSHL, is_q, vece, a0, a1, a2);
3307	}
3308	break;
3309	case INDEX_op_cmp_vec:
3310	{
3311	TCGCond cond = args[3];
3312	AArch64Insn insn;
3313
3314	switch (cond) {
3315	case TCG_COND_NE:
3316	if (const_args[2]) {
3317	if (is_scalar) {
3318	tcg_out_insn(s, rrr_e, CMTST, vece, a0, a1, a1);
3319	} else {
3320	tcg_out_insn(s, qrrr_e, CMTST, is_q, vece, a0, a1, a1);
3321	}
3322	} else {
3323	if (is_scalar) {
3324	tcg_out_insn(s, rrr_e, CMEQ, vece, a0, a1, a2);
3325	} else {
3326	tcg_out_insn(s, qrrr_e, CMEQ, is_q, vece, a0, a1, a2);
3327	}
3328	tcg_out_insn(s, qrr_e, NOT, is_q, 0, a0, a0);
3329	}
3330	break;
3331
3332	case TCG_COND_TSTNE:
3333	case TCG_COND_TSTEQ:
3334	if (const_args[2]) {
3335	/* (x & 0) == 0 */
3336	tcg_out_dupi_vec(s, type, MO_8, a0,
3337	-(cond == TCG_COND_TSTEQ));
3338	break;
3339	}
3340	if (is_scalar) {
3341	tcg_out_insn(s, rrr_e, CMTST, vece, a0, a1, a2);
3342	} else {
3343	tcg_out_insn(s, qrrr_e, CMTST, is_q, vece, a0, a1, a2);
3344	}
3345	if (cond == TCG_COND_TSTEQ) {
3346	tcg_out_insn(s, qrr_e, NOT, is_q, 0, a0, a0);
3347	}
3348	break;
3349
3350	default:
3351	if (const_args[2]) {
3352	if (is_scalar) {
3353	insn = cmp0_scalar_insn[cond];
3354	if (insn) {
3355	tcg_out_insn_simd_rr(s, insn, vece, a0, a1);
3356	break;
3357	}
3358	} else {
3359	insn = cmp0_vec_insn[cond];
3360	if (insn) {
3361	tcg_out_insn_qrr_e(s, insn, is_q, vece, a0, a1);
3362	break;
3363	}
3364	}
3365	tcg_out_dupi_vec(s, type, MO_8, TCG_VEC_TMP0, 0);
3366	a2 = TCG_VEC_TMP0;
3367	}
3368	if (is_scalar) {
3369	insn = cmp_scalar_insn[cond];
3370	if (insn == 0) {
3371	TCGArg t;
3372	t = a1, a1 = a2, a2 = t;
3373	cond = tcg_swap_cond(cond);
3374	insn = cmp_scalar_insn[cond];
3375	tcg_debug_assert(insn != 0);
3376	}
3377	tcg_out_insn_rrr_e(s, insn, vece, a0, a1, a2);
3378	} else {
3379	insn = cmp_vec_insn[cond];
3380	if (insn == 0) {
3381	TCGArg t;
3382	t = a1, a1 = a2, a2 = t;
3383	cond = tcg_swap_cond(cond);
3384	insn = cmp_vec_insn[cond];
3385	tcg_debug_assert(insn != 0);
3386	}
3387	tcg_out_insn_qrrr_e(s, insn, is_q, vece, a0, a1, a2);
3388	}
3389	break;
3390	}
3391	}
3392	break;
3393
3394	case INDEX_op_bitsel_vec:
3395	a3 = args[3];
3396	if (a0 == a3) {
3397	tcg_out_insn(s, qrrr_e, BIT, is_q, 0, a0, a2, a1);
3398	} else if (a0 == a2) {
3399	tcg_out_insn(s, qrrr_e, BIF, is_q, 0, a0, a3, a1);
3400	} else {
3401	if (a0 != a1) {
3402	tcg_out_mov(s, type, a0, a1);
3403	}
3404	tcg_out_insn(s, qrrr_e, BSL, is_q, 0, a0, a2, a3);
3405	}
3406	break;
3407
3408	case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
3409	case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
3410	default:
3411	g_assert_not_reached();
3412	}
3413	}
3414
3415	int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
3416	{
3417	switch (opc) {
3418	case INDEX_op_add_vec:
3419	case INDEX_op_sub_vec:
3420	case INDEX_op_and_vec:
3421	case INDEX_op_or_vec:
3422	case INDEX_op_xor_vec:
3423	case INDEX_op_andc_vec:
3424	case INDEX_op_orc_vec:
3425	case INDEX_op_neg_vec:
3426	case INDEX_op_abs_vec:
3427	case INDEX_op_not_vec:
3428	case INDEX_op_cmp_vec:
3429	case INDEX_op_shli_vec:
3430	case INDEX_op_shri_vec:
3431	case INDEX_op_sari_vec:
3432	case INDEX_op_ssadd_vec:
3433	case INDEX_op_sssub_vec:
3434	case INDEX_op_usadd_vec:
3435	case INDEX_op_ussub_vec:
3436	case INDEX_op_shlv_vec:
3437	case INDEX_op_bitsel_vec:
3438	return 1;
3439	case INDEX_op_rotli_vec:
3440	case INDEX_op_shrv_vec:
3441	case INDEX_op_sarv_vec:
3442	case INDEX_op_rotlv_vec:
3443	case INDEX_op_rotrv_vec:
3444	return -1;
3445	case INDEX_op_mul_vec:
3446	case INDEX_op_smax_vec:
3447	case INDEX_op_smin_vec:
3448	case INDEX_op_umax_vec:
3449	case INDEX_op_umin_vec:
3450	return vece < MO_64;
3451
3452	default:
3453	return 0;
3454	}
3455	}
3456
3457	void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
3458	TCGArg a0, ...)
3459	{
3460	va_list va;
3461	TCGv_vec v0, v1, v2, t1, t2, c1;
3462	TCGArg a2;
3463
3464	va_start(va, a0);
3465	v0 = temp_tcgv_vec(arg_temp(a0));
3466	v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3467	a2 = va_arg(va, TCGArg);
3468	va_end(va);
3469
3470	switch (opc) {
3471	case INDEX_op_rotli_vec:
3472	t1 = tcg_temp_new_vec(type);
3473	tcg_gen_shri_vec(vece, t1, v1, -a2 & ((8 << vece) - 1));
3474	vec_gen_4(INDEX_op_aa64_sli_vec, type, vece,
3475	tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(v1), a2);
3476	tcg_temp_free_vec(t1);
3477	break;
3478
3479	case INDEX_op_shrv_vec:
3480	case INDEX_op_sarv_vec:
3481	/* Right shifts are negative left shifts for AArch64. */
3482	v2 = temp_tcgv_vec(arg_temp(a2));
3483	t1 = tcg_temp_new_vec(type);
3484	tcg_gen_neg_vec(vece, t1, v2);
3485	opc = (opc == INDEX_op_shrv_vec
3486	? INDEX_op_shlv_vec : INDEX_op_aa64_sshl_vec);
3487	vec_gen_3(opc, type, vece, tcgv_vec_arg(v0),
3488	tcgv_vec_arg(v1), tcgv_vec_arg(t1));
3489	tcg_temp_free_vec(t1);
3490	break;
3491
3492	case INDEX_op_rotlv_vec:
3493	v2 = temp_tcgv_vec(arg_temp(a2));
3494	t1 = tcg_temp_new_vec(type);
3495	c1 = tcg_constant_vec(type, vece, 8 << vece);
3496	tcg_gen_sub_vec(vece, t1, v2, c1);
3497	/* Right shifts are negative left shifts for AArch64. */
3498	vec_gen_3(INDEX_op_shlv_vec, type, vece, tcgv_vec_arg(t1),
3499	tcgv_vec_arg(v1), tcgv_vec_arg(t1));
3500	vec_gen_3(INDEX_op_shlv_vec, type, vece, tcgv_vec_arg(v0),
3501	tcgv_vec_arg(v1), tcgv_vec_arg(v2));
3502	tcg_gen_or_vec(vece, v0, v0, t1);
3503	tcg_temp_free_vec(t1);
3504	break;
3505
3506	case INDEX_op_rotrv_vec:
3507	v2 = temp_tcgv_vec(arg_temp(a2));
3508	t1 = tcg_temp_new_vec(type);
3509	t2 = tcg_temp_new_vec(type);
3510	c1 = tcg_constant_vec(type, vece, 8 << vece);
3511	tcg_gen_neg_vec(vece, t1, v2);
3512	tcg_gen_sub_vec(vece, t2, c1, v2);
3513	/* Right shifts are negative left shifts for AArch64. */
3514	vec_gen_3(INDEX_op_shlv_vec, type, vece, tcgv_vec_arg(t1),
3515	tcgv_vec_arg(v1), tcgv_vec_arg(t1));
3516	vec_gen_3(INDEX_op_shlv_vec, type, vece, tcgv_vec_arg(t2),
3517	tcgv_vec_arg(v1), tcgv_vec_arg(t2));
3518	tcg_gen_or_vec(vece, v0, t1, t2);
3519	tcg_temp_free_vec(t1);
3520	tcg_temp_free_vec(t2);
3521	break;
3522
3523	default:
3524	g_assert_not_reached();
3525	}
3526	}
3527
3528	static TCGConstraintSetIndex
3529	tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags)
3530	{
3531	switch (op) {
3532	case INDEX_op_add_vec:
3533	case INDEX_op_sub_vec:
3534	case INDEX_op_mul_vec:
3535	case INDEX_op_xor_vec:
3536	case INDEX_op_ssadd_vec:
3537	case INDEX_op_sssub_vec:
3538	case INDEX_op_usadd_vec:
3539	case INDEX_op_ussub_vec:
3540	case INDEX_op_smax_vec:
3541	case INDEX_op_smin_vec:
3542	case INDEX_op_umax_vec:
3543	case INDEX_op_umin_vec:
3544	case INDEX_op_shlv_vec:
3545	case INDEX_op_shrv_vec:
3546	case INDEX_op_sarv_vec:
3547	case INDEX_op_aa64_sshl_vec:
3548	return C_O1_I2(w, w, w);
3549	case INDEX_op_not_vec:
3550	case INDEX_op_neg_vec:
3551	case INDEX_op_abs_vec:
3552	case INDEX_op_shli_vec:
3553	case INDEX_op_shri_vec:
3554	case INDEX_op_sari_vec:
3555	return C_O1_I1(w, w);
3556	case INDEX_op_ld_vec:
3557	case INDEX_op_dupm_vec:
3558	return C_O1_I1(w, r);
3559	case INDEX_op_st_vec:
3560	return C_O0_I2(w, r);
3561	case INDEX_op_dup_vec:
3562	return C_O1_I1(w, wr);
3563	case INDEX_op_or_vec:
3564	case INDEX_op_andc_vec:
3565	return C_O1_I2(w, w, wO);
3566	case INDEX_op_and_vec:
3567	case INDEX_op_orc_vec:
3568	return C_O1_I2(w, w, wN);
3569	case INDEX_op_cmp_vec:
3570	return C_O1_I2(w, w, wZ);
3571	case INDEX_op_bitsel_vec:
3572	return C_O1_I3(w, w, w, w);
3573	case INDEX_op_aa64_sli_vec:
3574	return C_O1_I2(w, 0, w);
3575
3576	default:
3577	return C_NotImplemented;
3578	}
3579	}
3580
3581	static void tcg_target_init(TCGContext *s)
3582	{
3583	tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffffu;
3584	tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffffu;
3585	tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
3586	tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
3587
3588	tcg_target_call_clobber_regs = -1ull;
3589	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X19);
3590	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X20);
3591	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X21);
3592	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X22);
3593	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X23);
3594	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X24);
3595	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X25);
3596	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X26);
3597	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X27);
3598	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X28);
3599	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_X29);
3600	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V8);
3601	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V9);
3602	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V10);
3603	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V11);
3604	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V12);
3605	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V13);
3606	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V14);
3607	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V15);
3608
3609	s->reserved_regs = 0;
3610	tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
3611	tcg_regset_set_reg(s->reserved_regs, TCG_REG_FP);
3612	tcg_regset_set_reg(s->reserved_regs, TCG_REG_X18); /* platform register */
3613	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP0);
3614	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1);
3615	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP2);
3616	tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP0);
3617	}
3618
3619	/* Saving pairs: (X19, X20) .. (X27, X28), (X29(fp), X30(lr)). */
3620	#define PUSH_SIZE ((30 - 19 + 1) * 8)
3621
3622	#define FRAME_SIZE \
3623	((PUSH_SIZE \
3624	+ TCG_STATIC_CALL_ARGS_SIZE \
3625	+ CPU_TEMP_BUF_NLONGS * sizeof(long) \
3626	+ TCG_TARGET_STACK_ALIGN - 1) \
3627	& ~(TCG_TARGET_STACK_ALIGN - 1))
3628
3629	/* We're expecting a 2 byte uleb128 encoded value. */
3630	QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
3631
3632	/* We're expecting to use a single ADDI insn. */
3633	QEMU_BUILD_BUG_ON(FRAME_SIZE - PUSH_SIZE > 0xfff);
3634
3635	static void tcg_target_qemu_prologue(TCGContext *s)
3636	{
3637	TCGReg r;
3638
3639	tcg_out_bti(s, BTI_C);
3640
3641	/* Push (FP, LR) and allocate space for all saved registers. */
3642	tcg_out_insn(s, ldstpair, STP, TCG_REG_FP, TCG_REG_LR,
3643	TCG_REG_SP, -PUSH_SIZE, 1, 1);
3644
3645	/* Set up frame pointer for canonical unwinding. */
3646	tcg_out_movr_sp(s, TCG_TYPE_I64, TCG_REG_FP, TCG_REG_SP);
3647
3648	/* Store callee-preserved regs x19..x28. */
3649	for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
3650	int ofs = (r - TCG_REG_X19 + 2) * 8;
3651	tcg_out_insn(s, ldstpair, STP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
3652	}
3653
3654	/* Make stack space for TCG locals. */
3655	tcg_out_insn(s, addsub_imm, SUBI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
3656	FRAME_SIZE - PUSH_SIZE);
3657
3658	/* Inform TCG about how to find TCG locals with register, offset, size. */
3659	tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
3660	CPU_TEMP_BUF_NLONGS * sizeof(long));
3661
3662	if (!tcg_use_softmmu) {
3663	/*
3664	* Note that XZR cannot be encoded in the address base register slot,
3665	* as that actually encodes SP. Depending on the guest, we may need
3666	* to zero-extend the guest address via the address index register slot,
3667	* therefore we need to load even a zero guest base into a register.
3668	*/
3669	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base);
3670	tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
3671	}
3672
3673	tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
3674	tcg_out_insn(s, bcond_reg, BR, tcg_target_call_iarg_regs[1]);
3675
3676	/*
3677	* Return path for goto_ptr. Set return value to 0, a-la exit_tb,
3678	* and fall through to the rest of the epilogue.
3679	*/
3680	tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
3681	tcg_out_bti(s, BTI_J);
3682	tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0);
3683
3684	/* TB epilogue */
3685	tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
3686	tcg_out_bti(s, BTI_J);
3687
3688	/* Remove TCG locals stack space. */
3689	tcg_out_insn(s, addsub_imm, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
3690	FRAME_SIZE - PUSH_SIZE);
3691
3692	/* Restore registers x19..x28. */
3693	for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
3694	int ofs = (r - TCG_REG_X19 + 2) * 8;
3695	tcg_out_insn(s, ldstpair, LDP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
3696	}
3697
3698	/* Pop (FP, LR), restore SP to previous frame. */
3699	tcg_out_insn(s, ldstpair, LDP, TCG_REG_FP, TCG_REG_LR,
3700	TCG_REG_SP, PUSH_SIZE, 0, 1);
3701	tcg_out_insn(s, bcond_reg, RET, TCG_REG_LR);
3702	}
3703
3704	static void tcg_out_tb_start(TCGContext *s)
3705	{
3706	tcg_out_bti(s, BTI_J);
3707	}
3708
3709	static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
3710	{
3711	int i;
3712	for (i = 0; i < count; ++i) {
3713	p[i] = NOP;
3714	}
3715	}
3716
3717	typedef struct {
3718	DebugFrameHeader h;
3719	uint8_t fde_def_cfa[4];
3720	uint8_t fde_reg_ofs[24];
3721	} DebugFrame;
3722
3723	#define ELF_HOST_MACHINE EM_AARCH64
3724
3725	static const DebugFrame debug_frame = {
3726	.h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
3727	.h.cie.id = -1,
3728	.h.cie.version = 1,
3729	.h.cie.code_align = 1,
3730	.h.cie.data_align = 0x78, /* sleb128 -8 */
3731	.h.cie.return_column = TCG_REG_LR,
3732
3733	/* Total FDE size does not include the "len" member. */
3734	.h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
3735
3736	.fde_def_cfa = {
3737	12, TCG_REG_SP, /* DW_CFA_def_cfa sp, ... */
3738	(FRAME_SIZE & 0x7f) \| 0x80, /* ... uleb128 FRAME_SIZE */
3739	(FRAME_SIZE >> 7)
3740	},
3741	.fde_reg_ofs = {
3742	0x80 + 28, 1, /* DW_CFA_offset, x28, -8 */
3743	0x80 + 27, 2, /* DW_CFA_offset, x27, -16 */
3744	0x80 + 26, 3, /* DW_CFA_offset, x26, -24 */
3745	0x80 + 25, 4, /* DW_CFA_offset, x25, -32 */
3746	0x80 + 24, 5, /* DW_CFA_offset, x24, -40 */
3747	0x80 + 23, 6, /* DW_CFA_offset, x23, -48 */
3748	0x80 + 22, 7, /* DW_CFA_offset, x22, -56 */
3749	0x80 + 21, 8, /* DW_CFA_offset, x21, -64 */
3750	0x80 + 20, 9, /* DW_CFA_offset, x20, -72 */
3751	0x80 + 19, 10, /* DW_CFA_offset, x1p, -80 */
3752	0x80 + 30, 11, /* DW_CFA_offset, lr, -88 */
3753	0x80 + 29, 12, /* DW_CFA_offset, fp, -96 */
3754	}
3755	};
3756
3757	void tcg_register_jit(const void *buf, size_t buf_size)
3758	{
3759	tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
3760	}