tcg-target.c.inc · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

inc 4,627 lines 141 KB

Raw

1	/*
2	* Tiny Code Generator for QEMU
3	*
4	* Copyright (c) 2008 Fabrice Bellard
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a copy
7	* of this software and associated documentation files (the "Software"), to deal
8	* in the Software without restriction, including without limitation the rights
9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10	* copies of the Software, and to permit persons to whom the Software is
11	* furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22	* THE SOFTWARE.
23	*/
24
25	/* Used for function call generation. */
26	#define TCG_TARGET_STACK_ALIGN 16
27	#if defined(_WIN64)
28	#define TCG_TARGET_CALL_STACK_OFFSET 32
29	#else
30	#define TCG_TARGET_CALL_STACK_OFFSET 0
31	#endif
32	#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL
33	#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL
34	#if defined(_WIN64)
35	# define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_BY_REF
36	# define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_BY_VEC
37	#else
38	# define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL
39	# define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL
40	#endif
41
42	#ifdef CONFIG_DEBUG_TCG
43	static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
44	"%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
45	"%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
46	"%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7",
47	"%xmm8", "%xmm9", "%xmm10", "%xmm11",
48	"%xmm12", "%xmm13", "%xmm14", "%xmm15",
49	};
50	#endif
51
52	static const int tcg_target_reg_alloc_order[] = {
53	TCG_REG_RBP,
54	TCG_REG_RBX,
55	TCG_REG_R12,
56	TCG_REG_R13,
57	TCG_REG_R14,
58	TCG_REG_R15,
59	TCG_REG_R10,
60	TCG_REG_R11,
61	TCG_REG_R9,
62	TCG_REG_R8,
63	TCG_REG_RCX,
64	TCG_REG_RDX,
65	TCG_REG_RSI,
66	TCG_REG_RDI,
67	TCG_REG_RAX,
68	TCG_REG_XMM0,
69	TCG_REG_XMM1,
70	TCG_REG_XMM2,
71	TCG_REG_XMM3,
72	TCG_REG_XMM4,
73	TCG_REG_XMM5,
74	#ifndef _WIN64
75	/* The Win64 ABI has xmm6-xmm15 as caller-saves, and we do not save
76	any of them. Therefore only allow xmm0-xmm5 to be allocated. */
77	TCG_REG_XMM6,
78	TCG_REG_XMM7,
79	TCG_REG_XMM8,
80	TCG_REG_XMM9,
81	TCG_REG_XMM10,
82	TCG_REG_XMM11,
83	TCG_REG_XMM12,
84	TCG_REG_XMM13,
85	TCG_REG_XMM14,
86	TCG_REG_XMM15,
87	#endif
88	};
89
90	#define TCG_TMP_VEC TCG_REG_XMM5
91
92	static const int tcg_target_call_iarg_regs[] = {
93	#if defined(_WIN64)
94	TCG_REG_RCX,
95	TCG_REG_RDX,
96	#else
97	TCG_REG_RDI,
98	TCG_REG_RSI,
99	TCG_REG_RDX,
100	TCG_REG_RCX,
101	#endif
102	TCG_REG_R8,
103	TCG_REG_R9,
104	};
105
106	static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot)
107	{
108	switch (kind) {
109	case TCG_CALL_RET_NORMAL:
110	tcg_debug_assert(slot >= 0 && slot <= 1);
111	return slot ? TCG_REG_EDX : TCG_REG_EAX;
112	#ifdef _WIN64
113	case TCG_CALL_RET_BY_VEC:
114	tcg_debug_assert(slot == 0);
115	return TCG_REG_XMM0;
116	#endif
117	default:
118	g_assert_not_reached();
119	}
120	}
121
122	/* Constants we accept. */
123	#define TCG_CT_CONST_S32 0x100
124	#define TCG_CT_CONST_U32 0x200
125	#define TCG_CT_CONST_I32 0x400
126	#define TCG_CT_CONST_WSZ 0x800
127	#define TCG_CT_CONST_TST 0x1000
128	#define TCG_CT_CONST_ZERO 0x2000
129
130	/* Registers used with L constraint. */
131	#define TCG_REG_L0 tcg_target_call_iarg_regs[0]
132	#define TCG_REG_L1 tcg_target_call_iarg_regs[1]
133
134	#define ALL_GENERAL_REGS 0x0000ffffu
135	#define ALL_VECTOR_REGS 0xffff0000u
136	#define ALL_BYTEL_REGS ALL_GENERAL_REGS
137	#define SOFTMMU_RESERVE_REGS \
138	(tcg_use_softmmu ? (1 << TCG_REG_L0) \| (1 << TCG_REG_L1) : 0)
139
140	#define have_bmi2 (cpuinfo & CPUINFO_BMI2)
141	#define have_lzcnt (cpuinfo & CPUINFO_LZCNT)
142
143	static const tcg_insn_unit *tb_ret_addr;
144
145	static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
146	intptr_t value, intptr_t addend)
147	{
148	value += addend;
149	switch (type) {
150	case R_386_PC32:
151	value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr);
152	if (value != (int32_t)value) {
153	return false;
154	}
155	tcg_patch32(code_ptr, value);
156	break;
157	case R_386_PC8:
158	value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr);
159	if (value != (int8_t)value) {
160	return false;
161	}
162	tcg_patch8(code_ptr, value);
163	break;
164	default:
165	g_assert_not_reached();
166	}
167	return true;
168	}
169
170	/* test if a constant matches the constraint */
171	static bool tcg_target_const_match(int64_t val, int ct,
172	TCGType type, TCGCond cond, int vece)
173	{
174	if (ct & TCG_CT_CONST) {
175	return 1;
176	}
177	if (type == TCG_TYPE_I32) {
178	if (ct & (TCG_CT_CONST_S32 \| TCG_CT_CONST_U32 \|
179	TCG_CT_CONST_I32 \| TCG_CT_CONST_TST)) {
180	return 1;
181	}
182	} else {
183	if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
184	return 1;
185	}
186	if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
187	return 1;
188	}
189	if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) {
190	return 1;
191	}
192	/*
193	* This will be used in combination with TCG_CT_CONST_S32,
194	* so "normal" TESTQ is already matched. Also accept:
195	* TESTQ -> TESTL (uint32_t)
196	* TESTQ -> BT (is_power_of_2)
197	*/
198	if ((ct & TCG_CT_CONST_TST)
199	&& is_tst_cond(cond)
200	&& (val == (uint32_t)val \|\| is_power_of_2(val))) {
201	return 1;
202	}
203	}
204	if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) {
205	return 1;
206	}
207	if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
208	return 1;
209	}
210	return 0;
211	}
212
213	# define LOWREGMASK(x) ((x) & 7)
214
215	#define P_EXT 0x100 /* 0x0f opcode prefix */
216	#define P_EXT38 0x200 /* 0x0f 0x38 opcode prefix */
217	#define P_DATA16 0x400 /* 0x66 opcode prefix */
218	#define P_VEXW 0x1000 /* Set VEX.W = 1 */
219	#define P_REXW P_VEXW /* Set REX.W = 1; match VEXW */
220	#define P_REXB_R 0x2000 /* REG field as byte register */
221	#define P_REXB_RM 0x4000 /* R/M field as byte register */
222	#define P_GS 0x8000 /* gs segment override */
223	#define P_EXT3A 0x10000 /* 0x0f 0x3a opcode prefix */
224	#define P_SIMDF3 0x20000 /* 0xf3 opcode prefix */
225	#define P_SIMDF2 0x40000 /* 0xf2 opcode prefix */
226	#define P_VEXL 0x80000 /* Set VEX.L = 1 */
227	#define P_EVEX 0x100000 /* Requires EVEX encoding */
228
229	#define OPC_ARITH_EbIb (0x80)
230	#define OPC_ARITH_EvIz (0x81)
231	#define OPC_ARITH_EvIb (0x83)
232	#define OPC_ARITH_GvEv (0x03) /* ... plus (ARITH_FOO << 3) */
233	#define OPC_ANDN (0xf2 \| P_EXT38)
234	#define OPC_ADD_GvEv (OPC_ARITH_GvEv \| (ARITH_ADD << 3))
235	#define OPC_AND_GvEv (OPC_ARITH_GvEv \| (ARITH_AND << 3))
236	#define OPC_BLENDPS (0x0c \| P_EXT3A \| P_DATA16)
237	#define OPC_BSF (0xbc \| P_EXT)
238	#define OPC_BSR (0xbd \| P_EXT)
239	#define OPC_BSWAP (0xc8 \| P_EXT)
240	#define OPC_CALL_Jz (0xe8)
241	#define OPC_CMOVCC (0x40 \| P_EXT) /* ... plus condition code */
242	#define OPC_CMP_GvEv (OPC_ARITH_GvEv \| (ARITH_CMP << 3))
243	#define OPC_DEC_r32 (0x48)
244	#define OPC_IMUL_GvEv (0xaf \| P_EXT)
245	#define OPC_IMUL_GvEvIb (0x6b)
246	#define OPC_IMUL_GvEvIz (0x69)
247	#define OPC_INC_r32 (0x40)
248	#define OPC_JCC_long (0x80 \| P_EXT) /* ... plus condition code */
249	#define OPC_JCC_short (0x70) /* ... plus condition code */
250	#define OPC_JMP_long (0xe9)
251	#define OPC_JMP_short (0xeb)
252	#define OPC_LEA (0x8d)
253	#define OPC_LZCNT (0xbd \| P_EXT \| P_SIMDF3)
254	#define OPC_MOVB_EvGv (0x88) /* stores, more or less */
255	#define OPC_MOVL_EvGv (0x89) /* stores, more or less */
256	#define OPC_MOVL_GvEv (0x8b) /* loads, more or less */
257	#define OPC_MOVB_EvIz (0xc6)
258	#define OPC_MOVL_EvIz (0xc7)
259	#define OPC_MOVB_Ib (0xb0)
260	#define OPC_MOVL_Iv (0xb8)
261	#define OPC_MOVBE_GyMy (0xf0 \| P_EXT38)
262	#define OPC_MOVBE_MyGy (0xf1 \| P_EXT38)
263	#define OPC_MOVD_VyEy (0x6e \| P_EXT \| P_DATA16)
264	#define OPC_MOVD_EyVy (0x7e \| P_EXT \| P_DATA16)
265	#define OPC_MOVDDUP (0x12 \| P_EXT \| P_SIMDF2)
266	#define OPC_MOVDQA_VxWx (0x6f \| P_EXT \| P_DATA16)
267	#define OPC_MOVDQA_WxVx (0x7f \| P_EXT \| P_DATA16)
268	#define OPC_MOVDQU_VxWx (0x6f \| P_EXT \| P_SIMDF3)
269	#define OPC_MOVDQU_WxVx (0x7f \| P_EXT \| P_SIMDF3)
270	#define OPC_MOVQ_VqWq (0x7e \| P_EXT \| P_SIMDF3)
271	#define OPC_MOVQ_WqVq (0xd6 \| P_EXT \| P_DATA16)
272	#define OPC_MOVSBL (0xbe \| P_EXT)
273	#define OPC_MOVSWL (0xbf \| P_EXT)
274	#define OPC_MOVSLQ (0x63 \| P_REXW)
275	#define OPC_MOVZBL (0xb6 \| P_EXT)
276	#define OPC_MOVZWL (0xb7 \| P_EXT)
277	#define OPC_PABSB (0x1c \| P_EXT38 \| P_DATA16)
278	#define OPC_PABSW (0x1d \| P_EXT38 \| P_DATA16)
279	#define OPC_PABSD (0x1e \| P_EXT38 \| P_DATA16)
280	#define OPC_VPABSQ (0x1f \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
281	#define OPC_PACKSSDW (0x6b \| P_EXT \| P_DATA16)
282	#define OPC_PACKSSWB (0x63 \| P_EXT \| P_DATA16)
283	#define OPC_PACKUSDW (0x2b \| P_EXT38 \| P_DATA16)
284	#define OPC_PACKUSWB (0x67 \| P_EXT \| P_DATA16)
285	#define OPC_PADDB (0xfc \| P_EXT \| P_DATA16)
286	#define OPC_PADDW (0xfd \| P_EXT \| P_DATA16)
287	#define OPC_PADDD (0xfe \| P_EXT \| P_DATA16)
288	#define OPC_PADDQ (0xd4 \| P_EXT \| P_DATA16)
289	#define OPC_PADDSB (0xec \| P_EXT \| P_DATA16)
290	#define OPC_PADDSW (0xed \| P_EXT \| P_DATA16)
291	#define OPC_PADDUB (0xdc \| P_EXT \| P_DATA16)
292	#define OPC_PADDUW (0xdd \| P_EXT \| P_DATA16)
293	#define OPC_PAND (0xdb \| P_EXT \| P_DATA16)
294	#define OPC_PANDN (0xdf \| P_EXT \| P_DATA16)
295	#define OPC_PBLENDW (0x0e \| P_EXT3A \| P_DATA16)
296	#define OPC_PCMPEQB (0x74 \| P_EXT \| P_DATA16)
297	#define OPC_PCMPEQW (0x75 \| P_EXT \| P_DATA16)
298	#define OPC_PCMPEQD (0x76 \| P_EXT \| P_DATA16)
299	#define OPC_PCMPEQQ (0x29 \| P_EXT38 \| P_DATA16)
300	#define OPC_PCMPGTB (0x64 \| P_EXT \| P_DATA16)
301	#define OPC_PCMPGTW (0x65 \| P_EXT \| P_DATA16)
302	#define OPC_PCMPGTD (0x66 \| P_EXT \| P_DATA16)
303	#define OPC_PCMPGTQ (0x37 \| P_EXT38 \| P_DATA16)
304	#define OPC_PEXTRD (0x16 \| P_EXT3A \| P_DATA16)
305	#define OPC_PINSRD (0x22 \| P_EXT3A \| P_DATA16)
306	#define OPC_PMAXSB (0x3c \| P_EXT38 \| P_DATA16)
307	#define OPC_PMAXSW (0xee \| P_EXT \| P_DATA16)
308	#define OPC_PMAXSD (0x3d \| P_EXT38 \| P_DATA16)
309	#define OPC_VPMAXSQ (0x3d \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
310	#define OPC_PMAXUB (0xde \| P_EXT \| P_DATA16)
311	#define OPC_PMAXUW (0x3e \| P_EXT38 \| P_DATA16)
312	#define OPC_PMAXUD (0x3f \| P_EXT38 \| P_DATA16)
313	#define OPC_VPMAXUQ (0x3f \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
314	#define OPC_PMINSB (0x38 \| P_EXT38 \| P_DATA16)
315	#define OPC_PMINSW (0xea \| P_EXT \| P_DATA16)
316	#define OPC_PMINSD (0x39 \| P_EXT38 \| P_DATA16)
317	#define OPC_VPMINSQ (0x39 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
318	#define OPC_PMINUB (0xda \| P_EXT \| P_DATA16)
319	#define OPC_PMINUW (0x3a \| P_EXT38 \| P_DATA16)
320	#define OPC_PMINUD (0x3b \| P_EXT38 \| P_DATA16)
321	#define OPC_VPMINUQ (0x3b \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
322	#define OPC_PMOVSXBW (0x20 \| P_EXT38 \| P_DATA16)
323	#define OPC_PMOVSXWD (0x23 \| P_EXT38 \| P_DATA16)
324	#define OPC_PMOVSXDQ (0x25 \| P_EXT38 \| P_DATA16)
325	#define OPC_PMOVZXBW (0x30 \| P_EXT38 \| P_DATA16)
326	#define OPC_PMOVZXWD (0x33 \| P_EXT38 \| P_DATA16)
327	#define OPC_PMOVZXDQ (0x35 \| P_EXT38 \| P_DATA16)
328	#define OPC_PMULLW (0xd5 \| P_EXT \| P_DATA16)
329	#define OPC_PMULLD (0x40 \| P_EXT38 \| P_DATA16)
330	#define OPC_VPMULLQ (0x40 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
331	#define OPC_POR (0xeb \| P_EXT \| P_DATA16)
332	#define OPC_PSHUFB (0x00 \| P_EXT38 \| P_DATA16)
333	#define OPC_PSHUFD (0x70 \| P_EXT \| P_DATA16)
334	#define OPC_PSHUFLW (0x70 \| P_EXT \| P_SIMDF2)
335	#define OPC_PSHUFHW (0x70 \| P_EXT \| P_SIMDF3)
336	#define OPC_PSHIFTW_Ib (0x71 \| P_EXT \| P_DATA16) /* /2 /6 /4 */
337	#define OPC_PSHIFTD_Ib (0x72 \| P_EXT \| P_DATA16) /* /1 /2 /6 /4 */
338	#define OPC_PSHIFTQ_Ib (0x73 \| P_EXT \| P_DATA16) /* /2 /6 /4 */
339	#define OPC_PSLLW (0xf1 \| P_EXT \| P_DATA16)
340	#define OPC_PSLLD (0xf2 \| P_EXT \| P_DATA16)
341	#define OPC_PSLLQ (0xf3 \| P_EXT \| P_DATA16)
342	#define OPC_PSRAW (0xe1 \| P_EXT \| P_DATA16)
343	#define OPC_PSRAD (0xe2 \| P_EXT \| P_DATA16)
344	#define OPC_VPSRAQ (0xe2 \| P_EXT \| P_DATA16 \| P_VEXW \| P_EVEX)
345	#define OPC_PSRLW (0xd1 \| P_EXT \| P_DATA16)
346	#define OPC_PSRLD (0xd2 \| P_EXT \| P_DATA16)
347	#define OPC_PSRLQ (0xd3 \| P_EXT \| P_DATA16)
348	#define OPC_PSUBB (0xf8 \| P_EXT \| P_DATA16)
349	#define OPC_PSUBW (0xf9 \| P_EXT \| P_DATA16)
350	#define OPC_PSUBD (0xfa \| P_EXT \| P_DATA16)
351	#define OPC_PSUBQ (0xfb \| P_EXT \| P_DATA16)
352	#define OPC_PSUBSB (0xe8 \| P_EXT \| P_DATA16)
353	#define OPC_PSUBSW (0xe9 \| P_EXT \| P_DATA16)
354	#define OPC_PSUBUB (0xd8 \| P_EXT \| P_DATA16)
355	#define OPC_PSUBUW (0xd9 \| P_EXT \| P_DATA16)
356	#define OPC_PUNPCKLBW (0x60 \| P_EXT \| P_DATA16)
357	#define OPC_PUNPCKLWD (0x61 \| P_EXT \| P_DATA16)
358	#define OPC_PUNPCKLDQ (0x62 \| P_EXT \| P_DATA16)
359	#define OPC_PUNPCKLQDQ (0x6c \| P_EXT \| P_DATA16)
360	#define OPC_PUNPCKHBW (0x68 \| P_EXT \| P_DATA16)
361	#define OPC_PUNPCKHWD (0x69 \| P_EXT \| P_DATA16)
362	#define OPC_PUNPCKHDQ (0x6a \| P_EXT \| P_DATA16)
363	#define OPC_PUNPCKHQDQ (0x6d \| P_EXT \| P_DATA16)
364	#define OPC_PXOR (0xef \| P_EXT \| P_DATA16)
365	#define OPC_POP_r32 (0x58)
366	#define OPC_POPCNT (0xb8 \| P_EXT \| P_SIMDF3)
367	#define OPC_PUSH_r32 (0x50)
368	#define OPC_PUSH_Iv (0x68)
369	#define OPC_PUSH_Ib (0x6a)
370	#define OPC_RET (0xc3)
371	#define OPC_SETCC (0x90 \| P_EXT \| P_REXB_RM) /* ... plus cc */
372	#define OPC_SHIFT_1 (0xd1)
373	#define OPC_SHIFT_Ib (0xc1)
374	#define OPC_SHIFT_cl (0xd3)
375	#define OPC_SARX (0xf7 \| P_EXT38 \| P_SIMDF3)
376	#define OPC_SHUFPS (0xc6 \| P_EXT)
377	#define OPC_SHLX (0xf7 \| P_EXT38 \| P_DATA16)
378	#define OPC_SHRX (0xf7 \| P_EXT38 \| P_SIMDF2)
379	#define OPC_SHRD_Ib (0xac \| P_EXT)
380	#define OPC_STC (0xf9)
381	#define OPC_TESTB (0x84)
382	#define OPC_TESTL (0x85)
383	#define OPC_TZCNT (0xbc \| P_EXT \| P_SIMDF3)
384	#define OPC_UD2 (0x0b \| P_EXT)
385	#define OPC_VPBLENDD (0x02 \| P_EXT3A \| P_DATA16)
386	#define OPC_VPBLENDVB (0x4c \| P_EXT3A \| P_DATA16)
387	#define OPC_VPBLENDMB (0x66 \| P_EXT38 \| P_DATA16 \| P_EVEX)
388	#define OPC_VPBLENDMW (0x66 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
389	#define OPC_VPBLENDMD (0x64 \| P_EXT38 \| P_DATA16 \| P_EVEX)
390	#define OPC_VPBLENDMQ (0x64 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
391	#define OPC_VPCMPB (0x3f \| P_EXT3A \| P_DATA16 \| P_EVEX)
392	#define OPC_VPCMPUB (0x3e \| P_EXT3A \| P_DATA16 \| P_EVEX)
393	#define OPC_VPCMPW (0x3f \| P_EXT3A \| P_DATA16 \| P_VEXW \| P_EVEX)
394	#define OPC_VPCMPUW (0x3e \| P_EXT3A \| P_DATA16 \| P_VEXW \| P_EVEX)
395	#define OPC_VPCMPD (0x1f \| P_EXT3A \| P_DATA16 \| P_EVEX)
396	#define OPC_VPCMPUD (0x1e \| P_EXT3A \| P_DATA16 \| P_EVEX)
397	#define OPC_VPCMPQ (0x1f \| P_EXT3A \| P_DATA16 \| P_VEXW \| P_EVEX)
398	#define OPC_VPCMPUQ (0x1e \| P_EXT3A \| P_DATA16 \| P_VEXW \| P_EVEX)
399	#define OPC_VPINSRB (0x20 \| P_EXT3A \| P_DATA16)
400	#define OPC_VPINSRW (0xc4 \| P_EXT \| P_DATA16)
401	#define OPC_VBROADCASTSS (0x18 \| P_EXT38 \| P_DATA16)
402	#define OPC_VBROADCASTSD (0x19 \| P_EXT38 \| P_DATA16)
403	#define OPC_VPBROADCASTB (0x78 \| P_EXT38 \| P_DATA16)
404	#define OPC_VPBROADCASTW (0x79 \| P_EXT38 \| P_DATA16)
405	#define OPC_VPBROADCASTD (0x58 \| P_EXT38 \| P_DATA16)
406	#define OPC_VPBROADCASTQ (0x59 \| P_EXT38 \| P_DATA16)
407	#define OPC_VGF2P8AFFINEQB (0xce \| P_EXT3A \| P_DATA16 \| P_VEXW)
408	#define OPC_VPMOVM2B (0x28 \| P_EXT38 \| P_SIMDF3 \| P_EVEX)
409	#define OPC_VPMOVM2W (0x28 \| P_EXT38 \| P_SIMDF3 \| P_VEXW \| P_EVEX)
410	#define OPC_VPMOVM2D (0x38 \| P_EXT38 \| P_SIMDF3 \| P_EVEX)
411	#define OPC_VPMOVM2Q (0x38 \| P_EXT38 \| P_SIMDF3 \| P_VEXW \| P_EVEX)
412	#define OPC_VPERMQ (0x00 \| P_EXT3A \| P_DATA16 \| P_VEXW)
413	#define OPC_VPERM2I128 (0x46 \| P_EXT3A \| P_DATA16 \| P_VEXL)
414	#define OPC_VPROLVD (0x15 \| P_EXT38 \| P_DATA16 \| P_EVEX)
415	#define OPC_VPROLVQ (0x15 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
416	#define OPC_VPRORVD (0x14 \| P_EXT38 \| P_DATA16 \| P_EVEX)
417	#define OPC_VPRORVQ (0x14 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
418	#define OPC_VPSHLDW (0x70 \| P_EXT3A \| P_DATA16 \| P_VEXW \| P_EVEX)
419	#define OPC_VPSHLDD (0x71 \| P_EXT3A \| P_DATA16 \| P_EVEX)
420	#define OPC_VPSHLDQ (0x71 \| P_EXT3A \| P_DATA16 \| P_VEXW \| P_EVEX)
421	#define OPC_VPSHLDVW (0x70 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
422	#define OPC_VPSHLDVD (0x71 \| P_EXT38 \| P_DATA16 \| P_EVEX)
423	#define OPC_VPSHLDVQ (0x71 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
424	#define OPC_VPSHRDVW (0x72 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
425	#define OPC_VPSHRDVD (0x73 \| P_EXT38 \| P_DATA16 \| P_EVEX)
426	#define OPC_VPSHRDVQ (0x73 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
427	#define OPC_VPSLLVW (0x12 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
428	#define OPC_VPSLLVD (0x47 \| P_EXT38 \| P_DATA16)
429	#define OPC_VPSLLVQ (0x47 \| P_EXT38 \| P_DATA16 \| P_VEXW)
430	#define OPC_VPSRAVW (0x11 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
431	#define OPC_VPSRAVD (0x46 \| P_EXT38 \| P_DATA16)
432	#define OPC_VPSRAVQ (0x46 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
433	#define OPC_VPSRLVW (0x10 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
434	#define OPC_VPSRLVD (0x45 \| P_EXT38 \| P_DATA16)
435	#define OPC_VPSRLVQ (0x45 \| P_EXT38 \| P_DATA16 \| P_VEXW)
436	#define OPC_VPTERNLOGQ (0x25 \| P_EXT3A \| P_DATA16 \| P_VEXW \| P_EVEX)
437	#define OPC_VPTESTMB (0x26 \| P_EXT38 \| P_DATA16 \| P_EVEX)
438	#define OPC_VPTESTMW (0x26 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
439	#define OPC_VPTESTMD (0x27 \| P_EXT38 \| P_DATA16 \| P_EVEX)
440	#define OPC_VPTESTMQ (0x27 \| P_EXT38 \| P_DATA16 \| P_VEXW \| P_EVEX)
441	#define OPC_VPTESTNMB (0x26 \| P_EXT38 \| P_SIMDF3 \| P_EVEX)
442	#define OPC_VPTESTNMW (0x26 \| P_EXT38 \| P_SIMDF3 \| P_VEXW \| P_EVEX)
443	#define OPC_VPTESTNMD (0x27 \| P_EXT38 \| P_SIMDF3 \| P_EVEX)
444	#define OPC_VPTESTNMQ (0x27 \| P_EXT38 \| P_SIMDF3 \| P_VEXW \| P_EVEX)
445	#define OPC_VZEROUPPER (0x77 \| P_EXT)
446	#define OPC_XCHG_ax_r32 (0x90)
447	#define OPC_XCHG_EvGv (0x87)
448
449	#define OPC_GRP3_Eb (0xf6)
450	#define OPC_GRP3_Ev (0xf7)
451	#define OPC_GRP5 (0xff)
452	#define OPC_GRP14 (0x73 \| P_EXT \| P_DATA16)
453	#define OPC_GRPBT (0xba \| P_EXT)
454
455	#define OPC_GRPBT_BT 4
456	#define OPC_GRPBT_BTS 5
457	#define OPC_GRPBT_BTR 6
458	#define OPC_GRPBT_BTC 7
459
460	/* Group 1 opcode extensions for 0x80-0x83.
461	These are also used as modifiers for OPC_ARITH. */
462	#define ARITH_ADD 0
463	#define ARITH_OR 1
464	#define ARITH_ADC 2
465	#define ARITH_SBB 3
466	#define ARITH_AND 4
467	#define ARITH_SUB 5
468	#define ARITH_XOR 6
469	#define ARITH_CMP 7
470
471	/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3. */
472	#define SHIFT_ROL 0
473	#define SHIFT_ROR 1
474	#define SHIFT_SHL 4
475	#define SHIFT_SHR 5
476	#define SHIFT_SAR 7
477
478	/* Group 3 opcode extensions for 0xf6, 0xf7. To be used with OPC_GRP3. */
479	#define EXT3_TESTi 0
480	#define EXT3_NOT 2
481	#define EXT3_NEG 3
482	#define EXT3_MUL 4
483	#define EXT3_IMUL 5
484	#define EXT3_DIV 6
485	#define EXT3_IDIV 7
486
487	/* Group 5 opcode extensions for 0xff. To be used with OPC_GRP5. */
488	#define EXT5_INC_Ev 0
489	#define EXT5_DEC_Ev 1
490	#define EXT5_CALLN_Ev 2
491	#define EXT5_JMPN_Ev 4
492
493	/* Condition codes to be added to OPC_JCC_{long,short}. */
494	#define JCC_JMP (-1)
495	#define JCC_JO 0x0
496	#define JCC_JNO 0x1
497	#define JCC_JB 0x2
498	#define JCC_JAE 0x3
499	#define JCC_JE 0x4
500	#define JCC_JNE 0x5
501	#define JCC_JBE 0x6
502	#define JCC_JA 0x7
503	#define JCC_JS 0x8
504	#define JCC_JNS 0x9
505	#define JCC_JP 0xa
506	#define JCC_JNP 0xb
507	#define JCC_JL 0xc
508	#define JCC_JGE 0xd
509	#define JCC_JLE 0xe
510	#define JCC_JG 0xf
511
512	static const uint8_t tcg_cond_to_jcc[] = {
513	[TCG_COND_EQ] = JCC_JE,
514	[TCG_COND_NE] = JCC_JNE,
515	[TCG_COND_LT] = JCC_JL,
516	[TCG_COND_GE] = JCC_JGE,
517	[TCG_COND_LE] = JCC_JLE,
518	[TCG_COND_GT] = JCC_JG,
519	[TCG_COND_LTU] = JCC_JB,
520	[TCG_COND_GEU] = JCC_JAE,
521	[TCG_COND_LEU] = JCC_JBE,
522	[TCG_COND_GTU] = JCC_JA,
523	[TCG_COND_TSTEQ] = JCC_JE,
524	[TCG_COND_TSTNE] = JCC_JNE,
525	};
526
527	static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
528	{
529	int rex;
530
531	if (opc & P_GS) {
532	tcg_out8(s, 0x65);
533	}
534	if (opc & P_DATA16) {
535	/* We should never be asking for both 16 and 64-bit operation. */
536	tcg_debug_assert((opc & P_REXW) == 0);
537	tcg_out8(s, 0x66);
538	}
539	if (opc & P_SIMDF3) {
540	tcg_out8(s, 0xf3);
541	} else if (opc & P_SIMDF2) {
542	tcg_out8(s, 0xf2);
543	}
544
545	rex = 0;
546	rex \|= (opc & P_REXW) ? 0x8 : 0x0; /* REX.W */
547	rex \|= (r & 8) >> 1; /* REX.R */
548	rex \|= (x & 8) >> 2; /* REX.X */
549	rex \|= (rm & 8) >> 3; /* REX.B */
550
551	/* P_REXB_{R,RM} indicates that the given register is the low byte.
552	For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,
553	as otherwise the encoding indicates %[abcd]h. Note that the values
554	that are ORed in merely indicate that the REX byte must be present;
555	those bits get discarded in output. */
556	rex \|= opc & (r >= 4 ? P_REXB_R : 0);
557	rex \|= opc & (rm >= 4 ? P_REXB_RM : 0);
558
559	if (rex) {
560	tcg_out8(s, (uint8_t)(rex \| 0x40));
561	}
562
563	if (opc & (P_EXT \| P_EXT38 \| P_EXT3A)) {
564	tcg_out8(s, 0x0f);
565	if (opc & P_EXT38) {
566	tcg_out8(s, 0x38);
567	} else if (opc & P_EXT3A) {
568	tcg_out8(s, 0x3a);
569	}
570	}
571
572	tcg_out8(s, opc);
573	}
574
575	static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
576	{
577	tcg_out_opc(s, opc, r, rm, 0);
578	tcg_out8(s, 0xc0 \| (LOWREGMASK(r) << 3) \| LOWREGMASK(rm));
579	}
580
581	static void tcg_out_vex_opc(TCGContext *s, int opc, int r, int v,
582	int rm, int index)
583	{
584	int tmp;
585
586	if (opc & P_GS) {
587	tcg_out8(s, 0x65);
588	}
589	/* Use the two byte form if possible, which cannot encode
590	VEX.W, VEX.B, VEX.X, or an m-mmmm field other than P_EXT. */
591	if ((opc & (P_EXT \| P_EXT38 \| P_EXT3A \| P_VEXW)) == P_EXT
592	&& ((rm \| index) & 8) == 0) {
593	/* Two byte VEX prefix. */
594	tcg_out8(s, 0xc5);
595
596	tmp = (r & 8 ? 0 : 0x80); /* VEX.R */
597	} else {
598	/* Three byte VEX prefix. */
599	tcg_out8(s, 0xc4);
600
601	/* VEX.m-mmmm */
602	if (opc & P_EXT3A) {
603	tmp = 3;
604	} else if (opc & P_EXT38) {
605	tmp = 2;
606	} else if (opc & P_EXT) {
607	tmp = 1;
608	} else {
609	g_assert_not_reached();
610	}
611	tmp \|= (r & 8 ? 0 : 0x80); /* VEX.R */
612	tmp \|= (index & 8 ? 0 : 0x40); /* VEX.X */
613	tmp \|= (rm & 8 ? 0 : 0x20); /* VEX.B */
614	tcg_out8(s, tmp);
615
616	tmp = (opc & P_VEXW ? 0x80 : 0); /* VEX.W */
617	}
618
619	tmp \|= (opc & P_VEXL ? 0x04 : 0); /* VEX.L */
620	/* VEX.pp */
621	if (opc & P_DATA16) {
622	tmp \|= 1; /* 0x66 */
623	} else if (opc & P_SIMDF3) {
624	tmp \|= 2; /* 0xf3 */
625	} else if (opc & P_SIMDF2) {
626	tmp \|= 3; /* 0xf2 */
627	}
628	tmp \|= (~v & 15) << 3; /* VEX.vvvv */
629	tcg_out8(s, tmp);
630	tcg_out8(s, opc);
631	}
632
633	static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v,
634	int rm, int index, int aaa, bool z)
635	{
636	/* The entire 4-byte evex prefix; with R' and V' set. */
637	uint32_t p = 0x08041062;
638	int mm, pp;
639
640	tcg_debug_assert(have_avx512vl);
641
642	/* EVEX.mm */
643	if (opc & P_EXT3A) {
644	mm = 3;
645	} else if (opc & P_EXT38) {
646	mm = 2;
647	} else if (opc & P_EXT) {
648	mm = 1;
649	} else {
650	g_assert_not_reached();
651	}
652
653	/* EVEX.pp */
654	if (opc & P_DATA16) {
655	pp = 1; /* 0x66 */
656	} else if (opc & P_SIMDF3) {
657	pp = 2; /* 0xf3 */
658	} else if (opc & P_SIMDF2) {
659	pp = 3; /* 0xf2 */
660	} else {
661	pp = 0;
662	}
663
664	p = deposit32(p, 8, 2, mm);
665	p = deposit32(p, 13, 1, (rm & 8) == 0); /* EVEX.RXB.B */
666	p = deposit32(p, 14, 1, (index & 8) == 0); /* EVEX.RXB.X */
667	p = deposit32(p, 15, 1, (r & 8) == 0); /* EVEX.RXB.R */
668	p = deposit32(p, 16, 2, pp);
669	p = deposit32(p, 19, 4, ~v);
670	p = deposit32(p, 23, 1, (opc & P_VEXW) != 0);
671	p = deposit32(p, 24, 3, aaa);
672	p = deposit32(p, 29, 2, (opc & P_VEXL) != 0);
673	p = deposit32(p, 31, 1, z);
674
675	tcg_out32(s, p);
676	tcg_out8(s, opc);
677	}
678
679	static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm)
680	{
681	if (opc & P_EVEX) {
682	tcg_out_evex_opc(s, opc, r, v, rm, 0, 0, false);
683	} else {
684	tcg_out_vex_opc(s, opc, r, v, rm, 0);
685	}
686	tcg_out8(s, 0xc0 \| (LOWREGMASK(r) << 3) \| LOWREGMASK(rm));
687	}
688
689	static void tcg_out_vex_modrm_type(TCGContext *s, int opc,
690	int r, int v, int rm, TCGType type)
691	{
692	if (type == TCG_TYPE_V256) {
693	opc \|= P_VEXL;
694	}
695	tcg_out_vex_modrm(s, opc, r, v, rm);
696	}
697
698	static void tcg_out_evex_modrm_type(TCGContext *s, int opc, int r, int v,
699	int rm, int aaa, bool z, TCGType type)
700	{
701	if (type == TCG_TYPE_V256) {
702	opc \|= P_VEXL;
703	}
704	tcg_out_evex_opc(s, opc, r, v, rm, 0, aaa, z);
705	tcg_out8(s, 0xc0 \| (LOWREGMASK(r) << 3) \| LOWREGMASK(rm));
706	}
707
708	/* Output an opcode with a full "rm + (index<<shift) + offset" address mode.
709	We handle either RM and INDEX missing with a negative value. In 64-bit
710	mode for absolute addresses, ~RM is the size of the immediate operand
711	that will follow the instruction. */
712
713	static void tcg_out_sib_offset(TCGContext *s, int r, int rm, int index,
714	int shift, intptr_t offset)
715	{
716	int mod, len;
717
718	if (index < 0 && rm < 0) {
719	/*
720	* Try for a rip-relative addressing mode. This has replaced
721	* the 32-bit-mode absolute addressing encoding.
722	*/
723	intptr_t pc = (intptr_t)s->code_ptr + 5 + ~rm;
724	intptr_t disp = offset - pc;
725	if (disp == (int32_t)disp) {
726	tcg_out8(s, (LOWREGMASK(r) << 3) \| 5);
727	tcg_out32(s, disp);
728	return;
729	}
730
731	/*
732	* Try for an absolute address encoding. This requires the
733	* use of the MODRM+SIB encoding and is therefore larger than
734	* rip-relative addressing.
735	*/
736	if (offset == (int32_t)offset) {
737	tcg_out8(s, (LOWREGMASK(r) << 3) \| 4);
738	tcg_out8(s, (4 << 3) \| 5);
739	tcg_out32(s, offset);
740	return;
741	}
742
743	/* ??? The memory isn't directly addressable. */
744	g_assert_not_reached();
745	}
746
747	/* Find the length of the immediate addend. Note that the encoding
748	that would be used for (%ebp) indicates absolute addressing. */
749	if (rm < 0) {
750	mod = 0, len = 4, rm = 5;
751	} else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) {
752	mod = 0, len = 0;
753	} else if (offset == (int8_t)offset) {
754	mod = 0x40, len = 1;
755	} else {
756	mod = 0x80, len = 4;
757	}
758
759	/* Use a single byte MODRM format if possible. Note that the encoding
760	that would be used for %esp is the escape to the two byte form. */
761	if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) {
762	/* Single byte MODRM format. */
763	tcg_out8(s, mod \| (LOWREGMASK(r) << 3) \| LOWREGMASK(rm));
764	} else {
765	/* Two byte MODRM+SIB format. */
766
767	/* Note that the encoding that would place %esp into the index
768	field indicates no index register. In 64-bit mode, the REX.X
769	bit counts, so %r12 can be used as the index. */
770	if (index < 0) {
771	index = 4;
772	} else {
773	tcg_debug_assert(index != TCG_REG_ESP);
774	}
775
776	tcg_out8(s, mod \| (LOWREGMASK(r) << 3) \| 4);
777	tcg_out8(s, (shift << 6) \| (LOWREGMASK(index) << 3) \| LOWREGMASK(rm));
778	}
779
780	if (len == 1) {
781	tcg_out8(s, offset);
782	} else if (len == 4) {
783	tcg_out32(s, offset);
784	}
785	}
786
787	static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
788	int index, int shift, intptr_t offset)
789	{
790	tcg_out_opc(s, opc, r, rm < 0 ? 0 : rm, index < 0 ? 0 : index);
791	tcg_out_sib_offset(s, r, rm, index, shift, offset);
792	}
793
794	static void tcg_out_vex_modrm_sib_offset(TCGContext *s, int opc, int r, int v,
795	int rm, int index, int shift,
796	intptr_t offset)
797	{
798	tcg_out_vex_opc(s, opc, r, v, rm < 0 ? 0 : rm, index < 0 ? 0 : index);
799	tcg_out_sib_offset(s, r, rm, index, shift, offset);
800	}
801
802	/* A simplification of the above with no index or shift. */
803	static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r,
804	int rm, intptr_t offset)
805	{
806	tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset);
807	}
808
809	static inline void tcg_out_vex_modrm_offset(TCGContext *s, int opc, int r,
810	int v, int rm, intptr_t offset)
811	{
812	tcg_out_vex_modrm_sib_offset(s, opc, r, v, rm, -1, 0, offset);
813	}
814
815	/* Output an opcode with an expected reference to the constant pool. */
816	static inline void tcg_out_modrm_pool(TCGContext *s, int opc, int r)
817	{
818	tcg_out_opc(s, opc, r, 0, 0);
819	/* Absolute for 32-bit, pc-relative for 64-bit. */
820	tcg_out8(s, LOWREGMASK(r) << 3 \| 5);
821	tcg_out32(s, 0);
822	}
823
824	/* Output an opcode with an expected reference to the constant pool. */
825	static inline void tcg_out_vex_modrm_pool(TCGContext *s, int opc, int r)
826	{
827	tcg_out_vex_opc(s, opc, r, 0, 0, 0);
828	/* Absolute for 32-bit, pc-relative for 64-bit. */
829	tcg_out8(s, LOWREGMASK(r) << 3 \| 5);
830	tcg_out32(s, 0);
831	}
832
833	/* Generate dest op= src. Uses the same ARITH_* codes as tgen_arithi. */
834	static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src)
835	{
836	/* Propagate an opcode prefix, such as P_REXW. */
837	int ext = subop & ~0x7;
838	subop &= 0x7;
839
840	tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src);
841	}
842
843	static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
844	{
845	int rexw = 0;
846
847	if (arg == ret) {
848	return true;
849	}
850	switch (type) {
851	case TCG_TYPE_I64:
852	rexw = P_REXW;
853	/* fallthru */
854	case TCG_TYPE_I32:
855	if (ret < 16) {
856	if (arg < 16) {
857	tcg_out_modrm(s, OPC_MOVL_GvEv + rexw, ret, arg);
858	} else {
859	tcg_out_vex_modrm(s, OPC_MOVD_EyVy + rexw, arg, 0, ret);
860	}
861	} else {
862	if (arg < 16) {
863	tcg_out_vex_modrm(s, OPC_MOVD_VyEy + rexw, ret, 0, arg);
864	} else {
865	tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg);
866	}
867	}
868	break;
869
870	case TCG_TYPE_V64:
871	tcg_debug_assert(ret >= 16 && arg >= 16);
872	tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg);
873	break;
874	case TCG_TYPE_V128:
875	tcg_debug_assert(ret >= 16 && arg >= 16);
876	tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx, ret, 0, arg);
877	break;
878	case TCG_TYPE_V256:
879	tcg_debug_assert(ret >= 16 && arg >= 16);
880	tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx \| P_VEXL, ret, 0, arg);
881	break;
882
883	default:
884	g_assert_not_reached();
885	}
886	return true;
887	}
888
889	static const int avx2_dup_insn[4] = {
890	OPC_VPBROADCASTB, OPC_VPBROADCASTW,
891	OPC_VPBROADCASTD, OPC_VPBROADCASTQ,
892	};
893
894	static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
895	TCGReg r, TCGReg a)
896	{
897	if (have_avx2) {
898	tcg_out_vex_modrm_type(s, avx2_dup_insn[vece], r, 0, a, type);
899	} else {
900	switch (vece) {
901	case MO_8:
902	/* ??? With zero in a register, use PSHUFB. */
903	tcg_out_vex_modrm(s, OPC_PUNPCKLBW, r, a, a);
904	a = r;
905	/* FALLTHRU */
906	case MO_16:
907	tcg_out_vex_modrm(s, OPC_PUNPCKLWD, r, a, a);
908	a = r;
909	/* FALLTHRU */
910	case MO_32:
911	tcg_out_vex_modrm(s, OPC_PSHUFD, r, 0, a);
912	/* imm8 operand: all output lanes selected from input lane 0. */
913	tcg_out8(s, 0);
914	break;
915	case MO_64:
916	tcg_out_vex_modrm(s, OPC_PUNPCKLQDQ, r, a, a);
917	break;
918	default:
919	g_assert_not_reached();
920	}
921	}
922	return true;
923	}
924
925	static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
926	TCGReg r, TCGReg base, intptr_t offset)
927	{
928	if (have_avx2) {
929	int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0);
930	tcg_out_vex_modrm_offset(s, avx2_dup_insn[vece] + vex_l,
931	r, 0, base, offset);
932	} else {
933	switch (vece) {
934	case MO_64:
935	tcg_out_vex_modrm_offset(s, OPC_MOVDDUP, r, 0, base, offset);
936	break;
937	case MO_32:
938	tcg_out_vex_modrm_offset(s, OPC_VBROADCASTSS, r, 0, base, offset);
939	break;
940	case MO_16:
941	tcg_out_vex_modrm_offset(s, OPC_VPINSRW, r, r, base, offset);
942	tcg_out8(s, 0); /* imm8 */
943	tcg_out_dup_vec(s, type, vece, r, r);
944	break;
945	case MO_8:
946	tcg_out_vex_modrm_offset(s, OPC_VPINSRB, r, r, base, offset);
947	tcg_out8(s, 0); /* imm8 */
948	tcg_out_dup_vec(s, type, vece, r, r);
949	break;
950	default:
951	g_assert_not_reached();
952	}
953	}
954	return true;
955	}
956
957	static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
958	TCGReg ret, int64_t arg)
959	{
960	int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0);
961
962	if (arg == 0) {
963	tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret);
964	return;
965	}
966	if (arg == -1) {
967	tcg_out_vex_modrm(s, OPC_PCMPEQB + vex_l, ret, ret, ret);
968	return;
969	}
970
971	if (type == TCG_TYPE_V64) {
972	tcg_out_vex_modrm_pool(s, OPC_MOVQ_VqWq, ret);
973	} else if (have_avx2) {
974	tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTQ + vex_l, ret);
975	} else {
976	tcg_out_vex_modrm_pool(s, OPC_MOVDDUP, ret);
977	}
978	new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4);
979	}
980
981	static void tcg_out_movi_vec(TCGContext *s, TCGType type,
982	TCGReg ret, tcg_target_long arg)
983	{
984	if (arg == 0) {
985	tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret);
986	return;
987	}
988	if (arg == -1) {
989	tcg_out_vex_modrm(s, OPC_PCMPEQB, ret, ret, ret);
990	return;
991	}
992
993	int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW);
994	tcg_out_vex_modrm_pool(s, OPC_MOVD_VyEy + rexw, ret);
995	new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4);
996	}
997
998	static void tcg_out_movi_int(TCGContext *s, TCGType type,
999	TCGReg ret, tcg_target_long arg)
1000	{
1001	tcg_target_long diff;
1002
1003	if (arg == 0 && !s->carry_live) {
1004	tgen_arithr(s, ARITH_XOR, ret, ret);
1005	return;
1006	}
1007	if (arg == (uint32_t)arg \|\| type == TCG_TYPE_I32) {
1008	tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0);
1009	tcg_out32(s, arg);
1010	return;
1011	}
1012	if (arg == (int32_t)arg) {
1013	tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret);
1014	tcg_out32(s, arg);
1015	return;
1016	}
1017
1018	/* Try a 7 byte pc-relative lea before the 10 byte movq. */
1019	diff = tcg_pcrel_diff(s, (const void *)arg) - 7;
1020	if (diff == (int32_t)diff) {
1021	tcg_out_opc(s, OPC_LEA \| P_REXW, ret, 0, 0);
1022	tcg_out8(s, (LOWREGMASK(ret) << 3) \| 5);
1023	tcg_out32(s, diff);
1024	return;
1025	}
1026
1027	tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0);
1028	tcg_out64(s, arg);
1029	}
1030
1031	static void tcg_out_movi(TCGContext *s, TCGType type,
1032	TCGReg ret, tcg_target_long arg)
1033	{
1034	switch (type) {
1035	case TCG_TYPE_I32:
1036	case TCG_TYPE_I64:
1037	if (ret < 16) {
1038	tcg_out_movi_int(s, type, ret, arg);
1039	} else {
1040	tcg_out_movi_vec(s, type, ret, arg);
1041	}
1042	break;
1043	default:
1044	g_assert_not_reached();
1045	}
1046	}
1047
1048	static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
1049	{
1050	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1051	tcg_out_modrm(s, OPC_XCHG_EvGv + rexw, r1, r2);
1052	return true;
1053	}
1054
1055	static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
1056	tcg_target_long imm)
1057	{
1058	/* This function is only used for passing structs by reference. */
1059	tcg_debug_assert(imm == (int32_t)imm);
1060	tcg_out_modrm_offset(s, OPC_LEA \| P_REXW, rd, rs, imm);
1061	}
1062
1063	static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)
1064	{
1065	if (val == (int8_t)val) {
1066	tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0);
1067	tcg_out8(s, val);
1068	} else if (val == (int32_t)val) {
1069	tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);
1070	tcg_out32(s, val);
1071	} else {
1072	g_assert_not_reached();
1073	}
1074	}
1075
1076	static void tcg_out_mb(TCGContext *s, unsigned a0)
1077	{
1078	/* Given the strength of x86 memory ordering, we only need care for
1079	store-load ordering. Experimentally, "lock orl $0,0(%esp)" is
1080	faster than "mfence", so don't bother with the sse insn. */
1081	if (a0 & TCG_MO_ST_LD) {
1082	tcg_out8(s, 0xf0);
1083	tcg_out_modrm_offset(s, OPC_ARITH_EvIb, ARITH_OR, TCG_REG_ESP, 0);
1084	tcg_out8(s, 0);
1085	}
1086	}
1087
1088	static inline void tcg_out_push(TCGContext *s, int reg)
1089	{
1090	tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0);
1091	}
1092
1093	static inline void tcg_out_pop(TCGContext *s, int reg)
1094	{
1095	tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0);
1096	}
1097
1098	static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
1099	TCGReg arg1, intptr_t arg2)
1100	{
1101	switch (type) {
1102	case TCG_TYPE_I32:
1103	if (ret < 16) {
1104	tcg_out_modrm_offset(s, OPC_MOVL_GvEv, ret, arg1, arg2);
1105	} else {
1106	tcg_out_vex_modrm_offset(s, OPC_MOVD_VyEy, ret, 0, arg1, arg2);
1107	}
1108	break;
1109	case TCG_TYPE_I64:
1110	if (ret < 16) {
1111	tcg_out_modrm_offset(s, OPC_MOVL_GvEv \| P_REXW, ret, arg1, arg2);
1112	break;
1113	}
1114	/* FALLTHRU */
1115	case TCG_TYPE_V64:
1116	/* There is no instruction that can validate 8-byte alignment. */
1117	tcg_debug_assert(ret >= 16);
1118	tcg_out_vex_modrm_offset(s, OPC_MOVQ_VqWq, ret, 0, arg1, arg2);
1119	break;
1120	case TCG_TYPE_V128:
1121	/*
1122	* The gvec infrastructure is asserts that v128 vector loads
1123	* and stores use a 16-byte aligned offset. Validate that the
1124	* final pointer is aligned by using an insn that will SIGSEGV.
1125	*/
1126	tcg_debug_assert(ret >= 16);
1127	tcg_out_vex_modrm_offset(s, OPC_MOVDQA_VxWx, ret, 0, arg1, arg2);
1128	break;
1129	case TCG_TYPE_V256:
1130	/*
1131	* The gvec infrastructure only requires 16-byte alignment,
1132	* so here we must use an unaligned load.
1133	*/
1134	tcg_debug_assert(ret >= 16);
1135	tcg_out_vex_modrm_offset(s, OPC_MOVDQU_VxWx \| P_VEXL,
1136	ret, 0, arg1, arg2);
1137	break;
1138	default:
1139	g_assert_not_reached();
1140	}
1141	}
1142
1143	static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
1144	TCGReg arg1, intptr_t arg2)
1145	{
1146	switch (type) {
1147	case TCG_TYPE_I32:
1148	if (arg < 16) {
1149	tcg_out_modrm_offset(s, OPC_MOVL_EvGv, arg, arg1, arg2);
1150	} else {
1151	tcg_out_vex_modrm_offset(s, OPC_MOVD_EyVy, arg, 0, arg1, arg2);
1152	}
1153	break;
1154	case TCG_TYPE_I64:
1155	if (arg < 16) {
1156	tcg_out_modrm_offset(s, OPC_MOVL_EvGv \| P_REXW, arg, arg1, arg2);
1157	break;
1158	}
1159	/* FALLTHRU */
1160	case TCG_TYPE_V64:
1161	/* There is no instruction that can validate 8-byte alignment. */
1162	tcg_debug_assert(arg >= 16);
1163	tcg_out_vex_modrm_offset(s, OPC_MOVQ_WqVq, arg, 0, arg1, arg2);
1164	break;
1165	case TCG_TYPE_V128:
1166	/*
1167	* The gvec infrastructure is asserts that v128 vector loads
1168	* and stores use a 16-byte aligned offset. Validate that the
1169	* final pointer is aligned by using an insn that will SIGSEGV.
1170	*
1171	* This specific instance is also used by TCG_CALL_RET_BY_VEC,
1172	* for _WIN64, which must have SSE2 but may not have AVX.
1173	*/
1174	tcg_debug_assert(arg >= 16);
1175	if (have_avx1) {
1176	tcg_out_vex_modrm_offset(s, OPC_MOVDQA_WxVx, arg, 0, arg1, arg2);
1177	} else {
1178	tcg_out_modrm_offset(s, OPC_MOVDQA_WxVx, arg, arg1, arg2);
1179	}
1180	break;
1181	case TCG_TYPE_V256:
1182	/*
1183	* The gvec infrastructure only requires 16-byte alignment,
1184	* so here we must use an unaligned store.
1185	*/
1186	tcg_debug_assert(arg >= 16);
1187	tcg_out_vex_modrm_offset(s, OPC_MOVDQU_WxVx \| P_VEXL,
1188	arg, 0, arg1, arg2);
1189	break;
1190	default:
1191	g_assert_not_reached();
1192	}
1193	}
1194
1195	static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1196	TCGReg base, intptr_t ofs)
1197	{
1198	int rexw = 0;
1199	if (type == TCG_TYPE_I64) {
1200	if (val != (int32_t)val) {
1201	return false;
1202	}
1203	rexw = P_REXW;
1204	} else if (type != TCG_TYPE_I32) {
1205	return false;
1206	}
1207	tcg_out_modrm_offset(s, OPC_MOVL_EvIz \| rexw, 0, base, ofs);
1208	tcg_out32(s, val);
1209	return true;
1210	}
1211
1212	static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count)
1213	{
1214	/* Propagate an opcode prefix, such as P_DATA16. */
1215	int ext = subopc & ~0x7;
1216	subopc &= 0x7;
1217
1218	if (count == 1) {
1219	tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg);
1220	} else {
1221	tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg);
1222	tcg_out8(s, count);
1223	}
1224	}
1225
1226	static inline void tcg_out_bswap32(TCGContext *s, int reg)
1227	{
1228	tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0);
1229	}
1230
1231	static inline void tcg_out_rolw_8(TCGContext *s, int reg)
1232	{
1233	tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);
1234	}
1235
1236	static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src)
1237	{
1238	tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
1239	}
1240
1241	static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1242	{
1243	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1244	tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
1245	}
1246
1247	static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src)
1248	{
1249	/* movzwl */
1250	tcg_out_modrm(s, OPC_MOVZWL, dest, src);
1251	}
1252
1253	static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1254	{
1255	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1256	/* movsw[lq] */
1257	tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
1258	}
1259
1260	static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
1261	{
1262	/* 32-bit mov zero extends. */
1263	tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
1264	}
1265
1266	static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
1267	{
1268	tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
1269	}
1270
1271	static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
1272	{
1273	tcg_out_ext32s(s, dest, src);
1274	}
1275
1276	static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
1277	{
1278	if (dest != src) {
1279	tcg_out_ext32u(s, dest, src);
1280	}
1281	}
1282
1283	static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg dest, TCGReg src)
1284	{
1285	tcg_out_ext32u(s, dest, src);
1286	}
1287
1288	static inline void tcg_out_bswap64(TCGContext *s, int reg)
1289	{
1290	tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
1291	}
1292
1293	static const TCGOutOpUnary outop_revbit8 = {
1294	.base.static_constraint = C_NotImplemented,
1295	};
1296
1297	static const TCGOutOpBswap outop_revbit32 = {
1298	.base.static_constraint = C_NotImplemented,
1299	};
1300
1301	static const TCGOutOpUnary outop_revbit64 = {
1302	.base.static_constraint = C_NotImplemented,
1303	};
1304
1305	static void tgen_arithi(TCGContext *s, int c, int r0,
1306	tcg_target_long val, int cf)
1307	{
1308	int rexw = c & -8;
1309
1310	c &= 7;
1311
1312	switch (c) {
1313	case ARITH_ADD:
1314	case ARITH_SUB:
1315	if (!cf) {
1316	/*
1317	* ??? While INC is 2 bytes shorter than ADDL $1, they also induce
1318	* partial flags update stalls on Pentium4 and are not recommended
1319	* by current Intel optimization manuals.
1320	*/
1321	if (val == 1 \|\| val == -1) {
1322	int is_inc = (c == ARITH_ADD) ^ (val < 0);
1323	/*
1324	* The single-byte increment encodings are re-tasked
1325	* as the REX prefixes. Use the MODRM encoding.
1326	*/
1327	tcg_out_modrm(s, OPC_GRP5 + rexw,
1328	(is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0);
1329	return;
1330	}
1331	if (val == 128) {
1332	/*
1333	* Facilitate using an 8-bit immediate. Carry is inverted
1334	* by this transformation, so do it only if cf == 0.
1335	*/
1336	c ^= ARITH_ADD ^ ARITH_SUB;
1337	val = -128;
1338	}
1339	}
1340	break;
1341
1342	case ARITH_AND:
1343	if (val == 0xffffffffu) {
1344	tcg_out_ext32u(s, r0, r0);
1345	return;
1346	}
1347	if (val == (uint32_t)val) {
1348	/* AND with no high bits set can use a 32-bit operation. */
1349	rexw = 0;
1350	}
1351	if (val == 0xffu) {
1352	tcg_out_ext8u(s, r0, r0);
1353	return;
1354	}
1355	if (val == 0xffffu) {
1356	tcg_out_ext16u(s, r0, r0);
1357	return;
1358	}
1359	break;
1360
1361	case ARITH_OR:
1362	case ARITH_XOR:
1363	if (val >= 0x80 && val <= 0xff) {
1364	tcg_out_modrm(s, OPC_ARITH_EbIb + P_REXB_RM, c, r0);
1365	tcg_out8(s, val);
1366	return;
1367	}
1368	break;
1369	}
1370
1371	if (val == (int8_t)val) {
1372	tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0);
1373	tcg_out8(s, val);
1374	return;
1375	}
1376	if (rexw == 0 \|\| val == (int32_t)val) {
1377	tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0);
1378	tcg_out32(s, val);
1379	return;
1380	}
1381
1382	g_assert_not_reached();
1383	}
1384
1385	static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
1386	{
1387	if (val != 0) {
1388	tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0);
1389	}
1390	}
1391
1392	/* Set SMALL to force a short forward branch. */
1393	static void tcg_out_jxx(TCGContext s, int opc, TCGLabel l, bool small)
1394	{
1395	int32_t val, val1;
1396
1397	if (l->has_value) {
1398	val = tcg_pcrel_diff(s, l->u.value_ptr);
1399	val1 = val - 2;
1400	if ((int8_t)val1 == val1) {
1401	if (opc == -1) {
1402	tcg_out8(s, OPC_JMP_short);
1403	} else {
1404	tcg_out8(s, OPC_JCC_short + opc);
1405	}
1406	tcg_out8(s, val1);
1407	} else {
1408	tcg_debug_assert(!small);
1409	if (opc == -1) {
1410	tcg_out8(s, OPC_JMP_long);
1411	tcg_out32(s, val - 5);
1412	} else {
1413	tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
1414	tcg_out32(s, val - 6);
1415	}
1416	}
1417	} else if (small) {
1418	if (opc == -1) {
1419	tcg_out8(s, OPC_JMP_short);
1420	} else {
1421	tcg_out8(s, OPC_JCC_short + opc);
1422	}
1423	tcg_out_reloc(s, s->code_ptr, R_386_PC8, l, -1);
1424	s->code_ptr += 1;
1425	} else {
1426	if (opc == -1) {
1427	tcg_out8(s, OPC_JMP_long);
1428	} else {
1429	tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
1430	}
1431	tcg_out_reloc(s, s->code_ptr, R_386_PC32, l, -4);
1432	s->code_ptr += 4;
1433	}
1434	}
1435
1436	static void tcg_out_br(TCGContext s, TCGLabel l)
1437	{
1438	tcg_out_jxx(s, JCC_JMP, l, 0);
1439	}
1440
1441	static int tcg_out_cmp(TCGContext *s, TCGCond cond, TCGArg arg1,
1442	TCGArg arg2, int const_arg2, int rexw)
1443	{
1444	int jz, js;
1445
1446	if (!is_tst_cond(cond)) {
1447	if (!const_arg2) {
1448	tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2);
1449	} else if (arg2 == 0) {
1450	tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1);
1451	} else {
1452	tcg_debug_assert(!rexw \|\| arg2 == (int32_t)arg2);
1453	tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0);
1454	}
1455	return tcg_cond_to_jcc[cond];
1456	}
1457
1458	jz = tcg_cond_to_jcc[cond];
1459	js = (cond == TCG_COND_TSTNE ? JCC_JS : JCC_JNS);
1460
1461	if (!const_arg2) {
1462	tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg2);
1463	return jz;
1464	}
1465
1466	if (arg2 <= 0xff) {
1467	if (arg2 == 0x80) {
1468	tcg_out_modrm(s, OPC_TESTB \| P_REXB_R, arg1, arg1);
1469	return js;
1470	}
1471	if (arg2 == 0xff) {
1472	tcg_out_modrm(s, OPC_TESTB \| P_REXB_R, arg1, arg1);
1473	return jz;
1474	}
1475	tcg_out_modrm(s, OPC_GRP3_Eb \| P_REXB_RM, EXT3_TESTi, arg1);
1476	tcg_out8(s, arg2);
1477	return jz;
1478	}
1479
1480	if ((arg2 & ~0xff00) == 0 && arg1 < 4) {
1481	if (arg2 == 0x8000) {
1482	tcg_out_modrm(s, OPC_TESTB, arg1 + 4, arg1 + 4);
1483	return js;
1484	}
1485	if (arg2 == 0xff00) {
1486	tcg_out_modrm(s, OPC_TESTB, arg1 + 4, arg1 + 4);
1487	return jz;
1488	}
1489	tcg_out_modrm(s, OPC_GRP3_Eb, EXT3_TESTi, arg1 + 4);
1490	tcg_out8(s, arg2 >> 8);
1491	return jz;
1492	}
1493
1494	if (arg2 == 0xffff) {
1495	tcg_out_modrm(s, OPC_TESTL \| P_DATA16, arg1, arg1);
1496	return jz;
1497	}
1498	if (arg2 == 0xffffffffu) {
1499	tcg_out_modrm(s, OPC_TESTL, arg1, arg1);
1500	return jz;
1501	}
1502
1503	if (is_power_of_2(rexw ? arg2 : (uint32_t)arg2)) {
1504	int jc = (cond == TCG_COND_TSTNE ? JCC_JB : JCC_JAE);
1505	int sh = ctz64(arg2);
1506
1507	rexw = (sh & 32 ? P_REXW : 0);
1508	if ((sh & 31) == 31) {
1509	tcg_out_modrm(s, OPC_TESTL \| rexw, arg1, arg1);
1510	return js;
1511	} else {
1512	tcg_out_modrm(s, OPC_GRPBT \| rexw, OPC_GRPBT_BT, arg1);
1513	tcg_out8(s, sh);
1514	return jc;
1515	}
1516	}
1517
1518	if (rexw) {
1519	if (arg2 == (uint32_t)arg2) {
1520	rexw = 0;
1521	} else {
1522	tcg_debug_assert(arg2 == (int32_t)arg2);
1523	}
1524	}
1525	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_TESTi, arg1);
1526	tcg_out32(s, arg2);
1527	return jz;
1528	}
1529
1530	static void tcg_out_brcond(TCGContext *s, int rexw, TCGCond cond,
1531	TCGArg arg1, TCGArg arg2, int const_arg2,
1532	TCGLabel *label, bool small)
1533	{
1534	int jcc = tcg_out_cmp(s, cond, arg1, arg2, const_arg2, rexw);
1535	tcg_out_jxx(s, jcc, label, small);
1536	}
1537
1538	static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond,
1539	TCGReg arg1, TCGReg arg2, TCGLabel *label)
1540	{
1541	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1542	tcg_out_brcond(s, rexw, cond, arg1, arg2, false, label, false);
1543	}
1544
1545	static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond cond,
1546	TCGReg arg1, tcg_target_long arg2, TCGLabel *label)
1547	{
1548	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1549	tcg_out_brcond(s, rexw, cond, arg1, arg2, true, label, false);
1550	}
1551
1552	static const TCGOutOpBrcond outop_brcond = {
1553	.base.static_constraint = C_O0_I2(r, reT),
1554	.out_rr = tgen_brcond,
1555	.out_ri = tgen_brcondi,
1556	};
1557
1558	static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
1559	TCGReg dest, TCGReg arg1, TCGArg arg2,
1560	bool const_arg2, bool neg)
1561	{
1562	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1563	int cmp_rexw = rexw;
1564	bool inv = false;
1565	bool cleared;
1566	int jcc;
1567
1568	switch (cond) {
1569	case TCG_COND_NE:
1570	inv = true;
1571	/* fall through */
1572	case TCG_COND_EQ:
1573	/* If arg2 is 0, convert to LTU/GEU vs 1. */
1574	if (const_arg2 && arg2 == 0) {
1575	arg2 = 1;
1576	goto do_ltu;
1577	}
1578	break;
1579
1580	case TCG_COND_TSTNE:
1581	inv = true;
1582	/* fall through */
1583	case TCG_COND_TSTEQ:
1584	/* If arg2 is -1, convert to LTU/GEU vs 1. */
1585	if (const_arg2 && arg2 == 0xffffffffu) {
1586	arg2 = 1;
1587	cmp_rexw = 0;
1588	goto do_ltu;
1589	}
1590	break;
1591
1592	case TCG_COND_LEU:
1593	inv = true;
1594	/* fall through */
1595	case TCG_COND_GTU:
1596	/* If arg2 is a register, swap for LTU/GEU. */
1597	if (!const_arg2) {
1598	TCGReg t = arg1;
1599	arg1 = arg2;
1600	arg2 = t;
1601	goto do_ltu;
1602	}
1603	break;
1604
1605	case TCG_COND_GEU:
1606	inv = true;
1607	/* fall through */
1608	case TCG_COND_LTU:
1609	do_ltu:
1610	/*
1611	* Relying on the carry bit, use SBB to produce -1 if LTU, 0 if GEU.
1612	* We can then use NEG or INC to produce the desired result.
1613	* This is always smaller than the SETCC expansion.
1614	*/
1615	tcg_out_cmp(s, TCG_COND_LTU, arg1, arg2, const_arg2, cmp_rexw);
1616
1617	/* X - X - C = -C = (C ? -1 : 0) */
1618	tgen_arithr(s, ARITH_SBB + (neg ? rexw : 0), dest, dest);
1619	if (inv && neg) {
1620	/* ~(C ? -1 : 0) = (C ? 0 : -1) */
1621	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, dest);
1622	} else if (inv) {
1623	/* (C ? -1 : 0) + 1 = (C ? 0 : 1) */
1624	tgen_arithi(s, ARITH_ADD, dest, 1, 0);
1625	} else if (!neg) {
1626	/* -(C ? -1 : 0) = (C ? 1 : 0) */
1627	tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_NEG, dest);
1628	}
1629	return;
1630
1631	case TCG_COND_GE:
1632	inv = true;
1633	/* fall through */
1634	case TCG_COND_LT:
1635	/* If arg2 is 0, extract the sign bit. */
1636	if (const_arg2 && arg2 == 0) {
1637	tcg_out_mov(s, type, dest, arg1);
1638	if (inv) {
1639	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, dest);
1640	}
1641	tcg_out_shifti(s, (neg ? SHIFT_SAR : SHIFT_SHR) + rexw,
1642	dest, rexw ? 63 : 31);
1643	return;
1644	}
1645	break;
1646
1647	default:
1648	break;
1649	}
1650
1651	/*
1652	* If dest does not overlap the inputs, clearing it first is preferred.
1653	* The XOR breaks any false dependency for the low-byte write to dest,
1654	* and is also one byte smaller than MOVZBL.
1655	*/
1656	cleared = false;
1657	if (dest != arg1 && (const_arg2 \|\| dest != arg2)) {
1658	tgen_arithr(s, ARITH_XOR, dest, dest);
1659	cleared = true;
1660	}
1661
1662	jcc = tcg_out_cmp(s, cond, arg1, arg2, const_arg2, cmp_rexw);
1663	tcg_out_modrm(s, OPC_SETCC \| jcc, 0, dest);
1664
1665	if (!cleared) {
1666	tcg_out_ext8u(s, dest, dest);
1667	}
1668	if (neg) {
1669	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, dest);
1670	}
1671	}
1672
1673	static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
1674	TCGReg dest, TCGReg arg1, TCGReg arg2)
1675	{
1676	tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, false);
1677	}
1678
1679	static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond,
1680	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
1681	{
1682	tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, false);
1683	}
1684
1685	static const TCGOutOpSetcond outop_setcond = {
1686	.base.static_constraint = C_O1_I2(q, r, reT),
1687	.out_rrr = tgen_setcond,
1688	.out_rri = tgen_setcondi,
1689	};
1690
1691	static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond,
1692	TCGReg dest, TCGReg arg1, TCGReg arg2)
1693	{
1694	tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, true);
1695	}
1696
1697	static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond,
1698	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
1699	{
1700	tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, true);
1701	}
1702
1703	static const TCGOutOpSetcond outop_negsetcond = {
1704	.base.static_constraint = C_O1_I2(q, r, reT),
1705	.out_rrr = tgen_negsetcond,
1706	.out_rri = tgen_negsetcondi,
1707	};
1708
1709	static void tcg_out_cmov(TCGContext *s, int jcc, int rexw,
1710	TCGReg dest, TCGReg v1)
1711	{
1712	tcg_out_modrm(s, OPC_CMOVCC \| jcc \| rexw, dest, v1);
1713	}
1714
1715	static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond,
1716	TCGReg dest, TCGReg c1, TCGArg c2, bool const_c2,
1717	TCGArg vt, bool const_vt,
1718	TCGArg vf, bool consf_vf)
1719	{
1720	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1721	int jcc = tcg_out_cmp(s, cond, c1, c2, const_c2, rexw);
1722	tcg_out_cmov(s, jcc, rexw, dest, vt);
1723	}
1724
1725	static const TCGOutOpMovcond outop_movcond = {
1726	.base.static_constraint = C_O1_I4(r, r, reT, r, 0),
1727	.out = tgen_movcond,
1728	};
1729
1730	static void tcg_out_branch(TCGContext s, int call, const tcg_insn_unit dest)
1731	{
1732	intptr_t disp = tcg_pcrel_diff(s, dest) - 5;
1733
1734	if (disp == (int32_t)disp) {
1735	tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0);
1736	tcg_out32(s, disp);
1737	} else {
1738	/* rip-relative addressing into the constant pool.
1739	This is 6 + 8 = 14 bytes, as compared to using an
1740	immediate load 10 + 6 = 16 bytes, plus we may
1741	be able to re-use the pool constant for more calls. */
1742	tcg_out_opc(s, OPC_GRP5, 0, 0, 0);
1743	tcg_out8(s, (call ? EXT5_CALLN_Ev : EXT5_JMPN_Ev) << 3 \| 5);
1744	new_pool_label(s, (uintptr_t)dest, R_386_PC32, s->code_ptr, -4);
1745	tcg_out32(s, 0);
1746	}
1747	}
1748
1749	static void tcg_out_call(TCGContext s, const tcg_insn_unit dest,
1750	const TCGHelperInfo *info)
1751	{
1752	tcg_out_branch(s, 1, dest);
1753	}
1754
1755	static void tcg_out_jmp(TCGContext s, const tcg_insn_unit dest)
1756	{
1757	tcg_out_branch(s, 0, dest);
1758	}
1759
1760	static void tcg_out_nopn(TCGContext *s, int n)
1761	{
1762	int i;
1763	/* Emit 1 or 2 operand size prefixes for the standard one byte nop,
1764	* "xchg %eax,%eax", forming "xchg %ax,%ax". All cores accept the
1765	* duplicate prefix, and all of the interesting recent cores can
1766	* decode and discard the duplicates in a single cycle.
1767	*/
1768	tcg_debug_assert(n >= 1);
1769	for (i = 1; i < n; ++i) {
1770	tcg_out8(s, 0x66);
1771	}
1772	tcg_out8(s, 0x90);
1773	}
1774
1775	typedef struct {
1776	TCGReg base;
1777	int index;
1778	int ofs;
1779	int seg;
1780	TCGAtomAlign aa;
1781	} HostAddress;
1782
1783	bool tcg_target_has_memory_bswap(MemOp memop)
1784	{
1785	TCGAtomAlign aa;
1786
1787	if (!have_movbe) {
1788	return false;
1789	}
1790	if ((memop & MO_SIZE) < MO_128) {
1791	return true;
1792	}
1793
1794	/*
1795	* Reject 16-byte memop with 16-byte atomicity, i.e. VMOVDQA,
1796	* but do allow a pair of 64-bit operations, i.e. MOVBEQ.
1797	*/
1798	aa = atom_and_align_for_opc(tcg_ctx, memop, MO_ATOM_WITHIN16, true);
1799	return aa.atom < MO_128;
1800	}
1801
1802	/*
1803	* Because x86_64 has xchg to handle addr/data register overlap, we have
1804	* placed all input arguments before we need might need a scratch reg.
1805	*
1806	* Even then, a scratch is only needed for l->raddr. Rather than expose
1807	* a general-purpose scratch when we don't actually know it's available,
1808	* use the ra_gen hook to load into RAX if needed.
1809	*/
1810	static TCGReg ldst_ra_gen(TCGContext s, const TCGLabelQemuLdst l, int arg)
1811	{
1812	if (arg < 0) {
1813	arg = TCG_REG_RAX;
1814	}
1815	tcg_out_movi(s, TCG_TYPE_PTR, arg, (uintptr_t)l->raddr);
1816	return arg;
1817	}
1818
1819	static const TCGLdstHelperParam ldst_helper_param = {
1820	.ra_gen = ldst_ra_gen
1821	};
1822
1823	static void tcg_out_vec_to_pair(TCGContext *s, TCGType type,
1824	TCGReg l, TCGReg h, TCGReg v)
1825	{
1826	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1827
1828	/* vpmov{d,q} %v, %l */
1829	tcg_out_vex_modrm(s, OPC_MOVD_EyVy + rexw, v, 0, l);
1830	/* vpextr{d,q} $1, %v, %h */
1831	tcg_out_vex_modrm(s, OPC_PEXTRD + rexw, v, 0, h);
1832	tcg_out8(s, 1);
1833	}
1834
1835	static void tcg_out_pair_to_vec(TCGContext *s, TCGType type,
1836	TCGReg v, TCGReg l, TCGReg h)
1837	{
1838	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
1839
1840	/* vmov{d,q} %l, %v */
1841	tcg_out_vex_modrm(s, OPC_MOVD_VyEy + rexw, v, 0, l);
1842	/* vpinsr{d,q} $1, %h, %v, %v */
1843	tcg_out_vex_modrm(s, OPC_PINSRD + rexw, v, v, h);
1844	tcg_out8(s, 1);
1845	}
1846
1847	/*
1848	* Generate code for the slow path for a load at the end of block
1849	*/
1850	static bool tcg_out_qemu_ld_slow_path(TCGContext s, TCGLabelQemuLdst l)
1851	{
1852	MemOp opc = get_memop(l->oi);
1853	tcg_insn_unit **label_ptr = &l->label_ptr[0];
1854
1855	/* resolve label address */
1856	tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
1857	if (label_ptr[1]) {
1858	tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
1859	}
1860
1861	tcg_out_ld_helper_args(s, l, &ldst_helper_param);
1862	tcg_out_branch(s, 1, qemu_ld_helpers[opc & MO_SIZE]);
1863	tcg_out_ld_helper_ret(s, l, false, &ldst_helper_param);
1864
1865	tcg_out_jmp(s, l->raddr);
1866	return true;
1867	}
1868
1869	/*
1870	* Generate code for the slow path for a store at the end of block
1871	*/
1872	static bool tcg_out_qemu_st_slow_path(TCGContext s, TCGLabelQemuLdst l)
1873	{
1874	MemOp opc = get_memop(l->oi);
1875	tcg_insn_unit **label_ptr = &l->label_ptr[0];
1876
1877	/* resolve label address */
1878	tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
1879	if (label_ptr[1]) {
1880	tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
1881	}
1882
1883	tcg_out_st_helper_args(s, l, &ldst_helper_param);
1884	tcg_out_branch(s, 1, qemu_st_helpers[opc & MO_SIZE]);
1885
1886	tcg_out_jmp(s, l->raddr);
1887	return true;
1888	}
1889
1890	#ifdef CONFIG_USER_ONLY
1891	static HostAddress x86_guest_base = {
1892	.index = -1
1893	};
1894
1895	#if defined(__linux__)
1896	# include <asm/prctl.h>
1897	# include <sys/prctl.h>
1898	int arch_prctl(int code, unsigned long addr);
1899	static inline int setup_guest_base_seg(void)
1900	{
1901	if (arch_prctl(ARCH_SET_GS, guest_base) == 0) {
1902	return P_GS;
1903	}
1904	return 0;
1905	}
1906	#define setup_guest_base_seg setup_guest_base_seg
1907	#elif defined (__FreeBSD__) \|\| defined (__FreeBSD_kernel__)
1908	# include <machine/sysarch.h>
1909	static inline int setup_guest_base_seg(void)
1910	{
1911	if (sysarch(AMD64_SET_GSBASE, &guest_base) == 0) {
1912	return P_GS;
1913	}
1914	return 0;
1915	}
1916	#define setup_guest_base_seg setup_guest_base_seg
1917	#endif
1918	#else
1919	# define x86_guest_base ((HostAddress )({ qemu_build_not_reached(); NULL; }))
1920	#endif /* CONFIG_USER_ONLY */
1921	#ifndef setup_guest_base_seg
1922	# define setup_guest_base_seg() 0
1923	#endif
1924
1925	#define MIN_TLB_MASK_TABLE_OFS INT_MIN
1926
1927	/*
1928	* For softmmu, perform the TLB load and compare.
1929	* For useronly, perform any required alignment tests.
1930	* In both cases, return a TCGLabelQemuLdst structure if the slow path
1931	* is required and fill in @h with the host address for the fast path.
1932	*/
1933	static TCGLabelQemuLdst prepare_host_addr(TCGContext s, HostAddress *h,
1934	TCGReg addr, MemOpIdx oi, bool is_ld)
1935	{
1936	TCGLabelQemuLdst *ldst = NULL;
1937	MemOp opc = get_memop(oi);
1938	MemOp s_bits = opc & MO_SIZE;
1939	unsigned a_mask;
1940
1941	if (tcg_use_softmmu) {
1942	h->index = TCG_REG_L0;
1943	h->ofs = 0;
1944	h->seg = 0;
1945	} else {
1946	*h = x86_guest_base;
1947	}
1948	h->base = addr;
1949	h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_WITHIN16, s_bits == MO_128);
1950	a_mask = (1 << h->aa.align) - 1;
1951
1952	if (tcg_use_softmmu) {
1953	int cmp_ofs = is_ld ? offsetof(CPUTLBEntry, addr_read)
1954	: offsetof(CPUTLBEntry, addr_write);
1955	TCGType ttype = TCG_TYPE_I32;
1956	TCGType tlbtype = TCG_TYPE_I32;
1957	int trexw = 0, hrexw = 0, tlbrexw = 0;
1958	unsigned mem_index = get_mmuidx(oi);
1959	unsigned s_mask = (1 << s_bits) - 1;
1960	int fast_ofs = tlb_mask_table_ofs(s, mem_index);
1961	int tlb_mask;
1962
1963	ldst = new_ldst_label(s);
1964	ldst->is_ld = is_ld;
1965	ldst->oi = oi;
1966	ldst->addr_reg = addr;
1967
1968	ttype = s->addr_type;
1969	trexw = (ttype == TCG_TYPE_I32 ? 0 : P_REXW);
1970	if (TCG_TYPE_PTR == TCG_TYPE_I64) {
1971	hrexw = P_REXW;
1972	tlbtype = TCG_TYPE_I64;
1973	tlbrexw = P_REXW;
1974	}
1975
1976	tcg_out_mov(s, tlbtype, TCG_REG_L0, addr);
1977	tcg_out_shifti(s, SHIFT_SHR + tlbrexw, TCG_REG_L0,
1978	TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1979
1980	tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, TCG_REG_L0, TCG_AREG0,
1981	fast_ofs + offsetof(CPUTLBDescFast, mask));
1982
1983	tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, TCG_REG_L0, TCG_AREG0,
1984	fast_ofs + offsetof(CPUTLBDescFast, table));
1985
1986	/*
1987	* If the required alignment is at least as large as the access,
1988	* simply copy the address and mask. For lesser alignments,
1989	* check that we don't cross pages for the complete access.
1990	*/
1991	if (a_mask >= s_mask) {
1992	tcg_out_mov(s, ttype, TCG_REG_L1, addr);
1993	} else {
1994	tcg_out_modrm_offset(s, OPC_LEA + trexw, TCG_REG_L1,
1995	addr, s_mask - a_mask);
1996	}
1997	tlb_mask = TARGET_PAGE_MASK \| a_mask;
1998	tgen_arithi(s, ARITH_AND + trexw, TCG_REG_L1, tlb_mask, 0);
1999
2000	/* cmp 0(TCG_REG_L0), TCG_REG_L1 */
2001	tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw,
2002	TCG_REG_L1, TCG_REG_L0, cmp_ofs);
2003
2004	/* jne slow_path */
2005	tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
2006	ldst->label_ptr[0] = s->code_ptr;
2007	s->code_ptr += 4;
2008
2009	/* TLB Hit. */
2010	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_L0, TCG_REG_L0,
2011	offsetof(CPUTLBEntry, addend));
2012	} else if (a_mask) {
2013	int jcc;
2014
2015	ldst = new_ldst_label(s);
2016	ldst->is_ld = is_ld;
2017	ldst->oi = oi;
2018	ldst->addr_reg = addr;
2019
2020	/* jne slow_path */
2021	jcc = tcg_out_cmp(s, TCG_COND_TSTNE, addr, a_mask, true, false);
2022	tcg_out_opc(s, OPC_JCC_long + jcc, 0, 0, 0);
2023	ldst->label_ptr[0] = s->code_ptr;
2024	s->code_ptr += 4;
2025	}
2026
2027	return ldst;
2028	}
2029
2030	static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
2031	HostAddress h, TCGType type, MemOp memop)
2032	{
2033	bool use_movbe = false;
2034	int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW);
2035	int movop = OPC_MOVL_GvEv;
2036
2037	/* Do big-endian loads with movbe. */
2038	if (memop & MO_BSWAP) {
2039	tcg_debug_assert(have_movbe);
2040	use_movbe = true;
2041	movop = OPC_MOVBE_GyMy;
2042	}
2043
2044	switch (memop & MO_SSIZE) {
2045	case MO_UB:
2046	tcg_out_modrm_sib_offset(s, OPC_MOVZBL + h.seg, datalo,
2047	h.base, h.index, 0, h.ofs);
2048	break;
2049	case MO_SB:
2050	tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + h.seg, datalo,
2051	h.base, h.index, 0, h.ofs);
2052	break;
2053	case MO_UW:
2054	if (use_movbe) {
2055	/* There is no extending movbe; only low 16-bits are modified. */
2056	if (datalo != h.base && datalo != h.index) {
2057	/* XOR breaks dependency chains. */
2058	tgen_arithr(s, ARITH_XOR, datalo, datalo);
2059	tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg,
2060	datalo, h.base, h.index, 0, h.ofs);
2061	} else {
2062	tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg,
2063	datalo, h.base, h.index, 0, h.ofs);
2064	tcg_out_ext16u(s, datalo, datalo);
2065	}
2066	} else {
2067	tcg_out_modrm_sib_offset(s, OPC_MOVZWL + h.seg, datalo,
2068	h.base, h.index, 0, h.ofs);
2069	}
2070	break;
2071	case MO_SW:
2072	if (use_movbe) {
2073	tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg,
2074	datalo, h.base, h.index, 0, h.ofs);
2075	tcg_out_ext16s(s, type, datalo, datalo);
2076	} else {
2077	tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + h.seg,
2078	datalo, h.base, h.index, 0, h.ofs);
2079	}
2080	break;
2081	case MO_UL:
2082	tcg_out_modrm_sib_offset(s, movop + h.seg, datalo,
2083	h.base, h.index, 0, h.ofs);
2084	break;
2085	case MO_SL:
2086	if (use_movbe) {
2087	tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + h.seg, datalo,
2088	h.base, h.index, 0, h.ofs);
2089	tcg_out_ext32s(s, datalo, datalo);
2090	} else {
2091	tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + h.seg, datalo,
2092	h.base, h.index, 0, h.ofs);
2093	}
2094	break;
2095	case MO_UQ:
2096	tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo,
2097	h.base, h.index, 0, h.ofs);
2098	break;
2099
2100	case MO_128:
2101	/*
2102	* Without 16-byte atomicity, use integer regs.
2103	* That is where we want the data, and it allows bswaps.
2104	*/
2105	if (h.aa.atom < MO_128) {
2106	if (use_movbe) {
2107	TCGReg t = datalo;
2108	datalo = datahi;
2109	datahi = t;
2110	}
2111	if (h.base == datalo \|\| h.index == datalo) {
2112	tcg_out_modrm_sib_offset(s, OPC_LEA + P_REXW, datahi,
2113	h.base, h.index, 0, h.ofs);
2114	tcg_out_modrm_offset(s, movop + P_REXW + h.seg,
2115	datalo, datahi, 0);
2116	tcg_out_modrm_offset(s, movop + P_REXW + h.seg,
2117	datahi, datahi, 8);
2118	} else {
2119	tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo,
2120	h.base, h.index, 0, h.ofs);
2121	tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datahi,
2122	h.base, h.index, 0, h.ofs + 8);
2123	}
2124	break;
2125	}
2126
2127	/*
2128	* With 16-byte atomicity, a vector load is required.
2129	* If we already have 16-byte alignment, then VMOVDQA always works.
2130	* Else if VMOVDQU has atomicity with dynamic alignment, use that.
2131	* Else use we require a runtime test for alignment for VMOVDQA;
2132	* use VMOVDQU on the unaligned nonatomic path for simplicity.
2133	*/
2134	if (h.aa.align >= MO_128) {
2135	tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_VxWx + h.seg,
2136	TCG_TMP_VEC, 0,
2137	h.base, h.index, 0, h.ofs);
2138	} else if (cpuinfo & CPUINFO_ATOMIC_VMOVDQU) {
2139	tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQU_VxWx + h.seg,
2140	TCG_TMP_VEC, 0,
2141	h.base, h.index, 0, h.ofs);
2142	} else {
2143	TCGLabel *l1 = gen_new_label();
2144	TCGLabel *l2 = gen_new_label();
2145	int jcc;
2146
2147	jcc = tcg_out_cmp(s, TCG_COND_TSTNE, h.base, 15, true, false);
2148	tcg_out_jxx(s, jcc, l1, true);
2149
2150	tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_VxWx + h.seg,
2151	TCG_TMP_VEC, 0,
2152	h.base, h.index, 0, h.ofs);
2153	tcg_out_jxx(s, JCC_JMP, l2, true);
2154
2155	tcg_out_label(s, l1);
2156	tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQU_VxWx + h.seg,
2157	TCG_TMP_VEC, 0,
2158	h.base, h.index, 0, h.ofs);
2159	tcg_out_label(s, l2);
2160	}
2161	tcg_out_vec_to_pair(s, TCG_TYPE_I64, datalo, datahi, TCG_TMP_VEC);
2162	break;
2163
2164	default:
2165	g_assert_not_reached();
2166	}
2167	}
2168
2169	static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data,
2170	TCGReg addr, MemOpIdx oi)
2171	{
2172	TCGLabelQemuLdst *ldst;
2173	HostAddress h;
2174
2175	ldst = prepare_host_addr(s, &h, addr, oi, true);
2176	tcg_out_qemu_ld_direct(s, data, -1, h, type, get_memop(oi));
2177
2178	if (ldst) {
2179	ldst->type = type;
2180	ldst->datalo_reg = data;
2181	ldst->datahi_reg = -1;
2182	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2183	}
2184	}
2185
2186	static const TCGOutOpQemuLdSt outop_qemu_ld = {
2187	.base.static_constraint = C_O1_I1(r, L),
2188	.out = tgen_qemu_ld,
2189	};
2190
2191	static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo,
2192	TCGReg datahi, TCGReg addr, MemOpIdx oi)
2193	{
2194	TCGLabelQemuLdst *ldst;
2195	HostAddress h;
2196
2197	ldst = prepare_host_addr(s, &h, addr, oi, true);
2198	tcg_out_qemu_ld_direct(s, datalo, datahi, h, type, get_memop(oi));
2199
2200	if (ldst) {
2201	ldst->type = type;
2202	ldst->datalo_reg = datalo;
2203	ldst->datahi_reg = datahi;
2204	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2205	}
2206	}
2207
2208	static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = {
2209	.base.static_constraint = C_O2_I1(r, r, L),
2210	.out = tgen_qemu_ld2,
2211	};
2212
2213	static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
2214	HostAddress h, MemOp memop)
2215	{
2216	bool use_movbe = false;
2217	int movop = OPC_MOVL_EvGv;
2218
2219	/*
2220	* Do big-endian stores with movbe or system-mode.
2221	* User-only without movbe will have its swapping done generically.
2222	*/
2223	if (memop & MO_BSWAP) {
2224	tcg_debug_assert(have_movbe);
2225	use_movbe = true;
2226	movop = OPC_MOVBE_MyGy;
2227	}
2228
2229	switch (memop & MO_SIZE) {
2230	case MO_8:
2231	tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + h.seg,
2232	datalo, h.base, h.index, 0, h.ofs);
2233	break;
2234	case MO_16:
2235	tcg_out_modrm_sib_offset(s, movop + P_DATA16 + h.seg, datalo,
2236	h.base, h.index, 0, h.ofs);
2237	break;
2238	case MO_32:
2239	tcg_out_modrm_sib_offset(s, movop + h.seg, datalo,
2240	h.base, h.index, 0, h.ofs);
2241	break;
2242	case MO_64:
2243	tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo,
2244	h.base, h.index, 0, h.ofs);
2245	break;
2246
2247	case MO_128:
2248	/*
2249	* Without 16-byte atomicity, use integer regs.
2250	* That is where we have the data, and it allows bswaps.
2251	*/
2252	if (h.aa.atom < MO_128) {
2253	if (use_movbe) {
2254	TCGReg t = datalo;
2255	datalo = datahi;
2256	datahi = t;
2257	}
2258	tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo,
2259	h.base, h.index, 0, h.ofs);
2260	tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datahi,
2261	h.base, h.index, 0, h.ofs + 8);
2262	break;
2263	}
2264
2265	/*
2266	* With 16-byte atomicity, a vector store is required.
2267	* If we already have 16-byte alignment, then VMOVDQA always works.
2268	* Else if VMOVDQU has atomicity with dynamic alignment, use that.
2269	* Else use we require a runtime test for alignment for VMOVDQA;
2270	* use VMOVDQU on the unaligned nonatomic path for simplicity.
2271	*/
2272	tcg_out_pair_to_vec(s, TCG_TYPE_I64, TCG_TMP_VEC, datalo, datahi);
2273	if (h.aa.align >= MO_128) {
2274	tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_WxVx + h.seg,
2275	TCG_TMP_VEC, 0,
2276	h.base, h.index, 0, h.ofs);
2277	} else if (cpuinfo & CPUINFO_ATOMIC_VMOVDQU) {
2278	tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQU_WxVx + h.seg,
2279	TCG_TMP_VEC, 0,
2280	h.base, h.index, 0, h.ofs);
2281	} else {
2282	TCGLabel *l1 = gen_new_label();
2283	TCGLabel *l2 = gen_new_label();
2284	int jcc;
2285
2286	jcc = tcg_out_cmp(s, TCG_COND_TSTNE, h.base, 15, true, false);
2287	tcg_out_jxx(s, jcc, l1, true);
2288
2289	tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_WxVx + h.seg,
2290	TCG_TMP_VEC, 0,
2291	h.base, h.index, 0, h.ofs);
2292	tcg_out_jxx(s, JCC_JMP, l2, true);
2293
2294	tcg_out_label(s, l1);
2295	tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQU_WxVx + h.seg,
2296	TCG_TMP_VEC, 0,
2297	h.base, h.index, 0, h.ofs);
2298	tcg_out_label(s, l2);
2299	}
2300	break;
2301
2302	default:
2303	g_assert_not_reached();
2304	}
2305	}
2306
2307	static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data,
2308	TCGReg addr, MemOpIdx oi)
2309	{
2310	TCGLabelQemuLdst *ldst;
2311	HostAddress h;
2312
2313	ldst = prepare_host_addr(s, &h, addr, oi, false);
2314	tcg_out_qemu_st_direct(s, data, -1, h, get_memop(oi));
2315
2316	if (ldst) {
2317	ldst->type = type;
2318	ldst->datalo_reg = data;
2319	ldst->datahi_reg = -1;
2320	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2321	}
2322	}
2323
2324	static const TCGOutOpQemuLdSt outop_qemu_st = {
2325	.base.static_constraint = C_O0_I2(L, L),
2326	.out = tgen_qemu_st,
2327	};
2328
2329	static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo,
2330	TCGReg datahi, TCGReg addr, MemOpIdx oi)
2331	{
2332	TCGLabelQemuLdst *ldst;
2333	HostAddress h;
2334
2335	ldst = prepare_host_addr(s, &h, addr, oi, false);
2336	tcg_out_qemu_st_direct(s, datalo, datahi, h, get_memop(oi));
2337
2338	if (ldst) {
2339	ldst->type = type;
2340	ldst->datalo_reg = datalo;
2341	ldst->datahi_reg = datahi;
2342	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2343	}
2344	}
2345
2346	static const TCGOutOpQemuLdSt2 outop_qemu_st2 = {
2347	.base.static_constraint = C_O0_I3(L, L, L),
2348	.out = tgen_qemu_st2,
2349	};
2350
2351	static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
2352	{
2353	/* Reuse the zeroing that exists for goto_ptr. */
2354	if (a0 == 0) {
2355	tcg_out_jmp(s, tcg_code_gen_epilogue);
2356	} else {
2357	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
2358	tcg_out_jmp(s, tb_ret_addr);
2359	}
2360	}
2361
2362	static void tcg_out_goto_tb(TCGContext *s, int which)
2363	{
2364	/*
2365	* Jump displacement must be aligned for atomic patching;
2366	* see if we need to add extra nops before jump
2367	*/
2368	int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
2369	if (gap != 1) {
2370	tcg_out_nopn(s, gap - 1);
2371	}
2372	tcg_out8(s, OPC_JMP_long); /* jmp im */
2373	set_jmp_insn_offset(s, which);
2374	tcg_out32(s, 0);
2375	set_jmp_reset_offset(s, which);
2376	}
2377
2378	static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0)
2379	{
2380	/* Jump to the given host address (could be epilogue) */
2381	tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0);
2382	}
2383
2384	void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
2385	uintptr_t jmp_rx, uintptr_t jmp_rw)
2386	{
2387	/* patch the branch destination */
2388	uintptr_t addr = tb->jmp_target_addr[n];
2389	qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
2390	/* no need to flush icache explicitly */
2391	}
2392
2393
2394	static void tgen_add(TCGContext *s, TCGType type,
2395	TCGReg a0, TCGReg a1, TCGReg a2)
2396	{
2397	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2398
2399	if (a0 == a1) {
2400	tgen_arithr(s, ARITH_ADD + rexw, a0, a2);
2401	} else if (a0 == a2) {
2402	tgen_arithr(s, ARITH_ADD + rexw, a0, a1);
2403	} else {
2404	tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, 0);
2405	}
2406	}
2407
2408	static void tgen_addi(TCGContext *s, TCGType type,
2409	TCGReg a0, TCGReg a1, tcg_target_long a2)
2410	{
2411	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2412
2413	if (a0 == a1) {
2414	tgen_arithi(s, ARITH_ADD + rexw, a0, a2, false);
2415	} else {
2416	tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, -1, 0, a2);
2417	}
2418	}
2419
2420	static const TCGOutOpBinary outop_add = {
2421	.base.static_constraint = C_O1_I2(r, r, re),
2422	.out_rrr = tgen_add,
2423	.out_rri = tgen_addi,
2424	};
2425
2426	static void tgen_addco(TCGContext *s, TCGType type,
2427	TCGReg a0, TCGReg a1, TCGReg a2)
2428	{
2429	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2430	tgen_arithr(s, ARITH_ADD + rexw, a0, a2);
2431	}
2432
2433	static void tgen_addco_imm(TCGContext *s, TCGType type,
2434	TCGReg a0, TCGReg a1, tcg_target_long a2)
2435	{
2436	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2437	tgen_arithi(s, ARITH_ADD + rexw, a0, a2, true);
2438	}
2439
2440	static const TCGOutOpBinary outop_addco = {
2441	.base.static_constraint = C_O1_I2(r, 0, re),
2442	.out_rrr = tgen_addco,
2443	.out_rri = tgen_addco_imm,
2444	};
2445
2446	static void tgen_addcio(TCGContext *s, TCGType type,
2447	TCGReg a0, TCGReg a1, TCGReg a2)
2448	{
2449	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2450	tgen_arithr(s, ARITH_ADC + rexw, a0, a2);
2451	}
2452
2453	static void tgen_addcio_imm(TCGContext *s, TCGType type,
2454	TCGReg a0, TCGReg a1, tcg_target_long a2)
2455	{
2456	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2457	tgen_arithi(s, ARITH_ADC + rexw, a0, a2, true);
2458	}
2459
2460	static const TCGOutOpBinary outop_addcio = {
2461	.base.static_constraint = C_O1_I2(r, 0, re),
2462	.out_rrr = tgen_addcio,
2463	.out_rri = tgen_addcio_imm,
2464	};
2465
2466	static void tgen_addci_rrr(TCGContext *s, TCGType type,
2467	TCGReg a0, TCGReg a1, TCGReg a2)
2468	{
2469	/* Because "0O" is not a valid constraint, we must match ourselves. */
2470	if (a0 == a2) {
2471	tgen_addcio(s, type, a0, a0, a1);
2472	} else {
2473	tcg_out_mov(s, type, a0, a1);
2474	tgen_addcio(s, type, a0, a0, a2);
2475	}
2476	}
2477
2478	static void tgen_addci_rri(TCGContext *s, TCGType type,
2479	TCGReg a0, TCGReg a1, tcg_target_long a2)
2480	{
2481	tcg_out_mov(s, type, a0, a1);
2482	tgen_addcio_imm(s, type, a0, a0, a2);
2483	}
2484
2485	static void tgen_addci_rir(TCGContext *s, TCGType type,
2486	TCGReg a0, tcg_target_long a1, TCGReg a2)
2487	{
2488	tgen_addci_rri(s, type, a0, a2, a1);
2489	}
2490
2491	static void tgen_addci_rii(TCGContext *s, TCGType type, TCGReg a0,
2492	tcg_target_long a1, tcg_target_long a2)
2493	{
2494	if (a2 == 0) {
2495	/* Implement 0 + 0 + C with -(x - x - c). */
2496	tgen_arithr(s, ARITH_SBB, a0, a0);
2497	tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_NEG, a0);
2498	} else {
2499	tcg_out_movi(s, type, a0, a2);
2500	tgen_addcio_imm(s, type, a0, a0, a1);
2501	}
2502	}
2503
2504	static const TCGOutOpAddSubCarry outop_addci = {
2505	.base.static_constraint = C_O1_I2(r, rO, re),
2506	.out_rrr = tgen_addci_rrr,
2507	.out_rri = tgen_addci_rri,
2508	.out_rir = tgen_addci_rir,
2509	.out_rii = tgen_addci_rii,
2510	};
2511
2512	static void tcg_out_set_carry(TCGContext *s)
2513	{
2514	tcg_out8(s, OPC_STC);
2515	}
2516
2517	static void tgen_and(TCGContext *s, TCGType type,
2518	TCGReg a0, TCGReg a1, TCGReg a2)
2519	{
2520	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2521	tgen_arithr(s, ARITH_AND + rexw, a0, a2);
2522	}
2523
2524	static void tgen_andi(TCGContext *s, TCGType type,
2525	TCGReg a0, TCGReg a1, tcg_target_long a2)
2526	{
2527	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2528	tgen_arithi(s, ARITH_AND + rexw, a0, a2, false);
2529	}
2530
2531	static const TCGOutOpBinary outop_and = {
2532	.base.static_constraint = C_O1_I2(r, 0, reZ),
2533	.out_rrr = tgen_and,
2534	.out_rri = tgen_andi,
2535	};
2536
2537	static void tgen_andc(TCGContext *s, TCGType type,
2538	TCGReg a0, TCGReg a1, TCGReg a2)
2539	{
2540	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2541	tcg_out_vex_modrm(s, OPC_ANDN + rexw, a0, a2, a1);
2542	}
2543
2544	static TCGConstraintSetIndex cset_andc(TCGType type, unsigned flags)
2545	{
2546	return have_bmi1 ? C_O1_I2(r, r, r) : C_NotImplemented;
2547	}
2548
2549	static const TCGOutOpBinary outop_andc = {
2550	.base.static_constraint = C_Dynamic,
2551	.base.dynamic_constraint = cset_andc,
2552	.out_rrr = tgen_andc,
2553	};
2554
2555	static void tgen_clz(TCGContext *s, TCGType type,
2556	TCGReg a0, TCGReg a1, TCGReg a2)
2557	{
2558	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2559	int jcc;
2560
2561	if (have_lzcnt) {
2562	tcg_out_modrm(s, OPC_LZCNT + rexw, a0, a1);
2563	jcc = JCC_JB;
2564	} else {
2565	/* Recall that the output of BSR is the index not the count. */
2566	tcg_out_modrm(s, OPC_BSR + rexw, a0, a1);
2567	tgen_arithi(s, ARITH_XOR + rexw, a0, rexw ? 63 : 31, 0);
2568
2569	/* Since we have destroyed the flags from BSR, we have to re-test. */
2570	jcc = tcg_out_cmp(s, TCG_COND_EQ, a1, 0, 1, rexw);
2571	}
2572	tcg_out_cmov(s, jcc, rexw, a0, a2);
2573	}
2574
2575	static void tgen_clzi(TCGContext *s, TCGType type,
2576	TCGReg a0, TCGReg a1, tcg_target_long a2)
2577	{
2578	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2579	tcg_out_modrm(s, OPC_LZCNT + rexw, a0, a1);
2580	}
2581
2582	static TCGConstraintSetIndex cset_clz(TCGType type, unsigned flags)
2583	{
2584	return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r);
2585	}
2586
2587	static const TCGOutOpBinary outop_clz = {
2588	.base.static_constraint = C_Dynamic,
2589	.base.dynamic_constraint = cset_clz,
2590	.out_rrr = tgen_clz,
2591	.out_rri = tgen_clzi,
2592	};
2593
2594	static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2595	{
2596	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2597	tcg_out_modrm(s, OPC_POPCNT + rexw, a0, a1);
2598	}
2599
2600	static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags)
2601	{
2602	return have_popcnt ? C_O1_I1(r, r) : C_NotImplemented;
2603	}
2604
2605	static const TCGOutOpUnary outop_ctpop = {
2606	.base.static_constraint = C_Dynamic,
2607	.base.dynamic_constraint = cset_ctpop,
2608	.out_rr = tgen_ctpop,
2609	};
2610
2611	static void tgen_ctz(TCGContext *s, TCGType type,
2612	TCGReg a0, TCGReg a1, TCGReg a2)
2613	{
2614	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2615	int jcc;
2616
2617	if (have_bmi1) {
2618	tcg_out_modrm(s, OPC_TZCNT + rexw, a0, a1);
2619	jcc = JCC_JB;
2620	} else {
2621	tcg_out_modrm(s, OPC_BSF + rexw, a0, a1);
2622	jcc = JCC_JE;
2623	}
2624	tcg_out_cmov(s, jcc, rexw, a0, a2);
2625	}
2626
2627	static void tgen_ctzi(TCGContext *s, TCGType type,
2628	TCGReg a0, TCGReg a1, tcg_target_long a2)
2629	{
2630	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2631	tcg_out_modrm(s, OPC_TZCNT + rexw, a0, a1);
2632	}
2633
2634	static TCGConstraintSetIndex cset_ctz(TCGType type, unsigned flags)
2635	{
2636	return have_bmi1 ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r);
2637	}
2638
2639	static const TCGOutOpBinary outop_ctz = {
2640	.base.static_constraint = C_Dynamic,
2641	.base.dynamic_constraint = cset_ctz,
2642	.out_rrr = tgen_ctz,
2643	.out_rri = tgen_ctzi,
2644	};
2645
2646	static const TCGOutOpBinary outop_divs = {
2647	.base.static_constraint = C_NotImplemented,
2648	};
2649
2650	static void tgen_divs2(TCGContext *s, TCGType type,
2651	TCGReg a0, TCGReg a1, TCGReg a4)
2652	{
2653	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2654	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, a4);
2655	}
2656
2657	static const TCGOutOpDivRem outop_divs2 = {
2658	.base.static_constraint = C_O2_I3(a, d, 0, 1, r),
2659	.out_rr01r = tgen_divs2,
2660	};
2661
2662	static const TCGOutOpBinary outop_divu = {
2663	.base.static_constraint = C_NotImplemented,
2664	};
2665
2666	static void tgen_divu2(TCGContext *s, TCGType type,
2667	TCGReg a0, TCGReg a1, TCGReg a4)
2668	{
2669	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2670	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, a4);
2671	}
2672
2673	static const TCGOutOpDivRem outop_divu2 = {
2674	.base.static_constraint = C_O2_I3(a, d, 0, 1, r),
2675	.out_rr01r = tgen_divu2,
2676	};
2677
2678	static const TCGOutOpBinary outop_eqv = {
2679	.base.static_constraint = C_NotImplemented,
2680	};
2681
2682	static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
2683	{
2684	tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32);
2685	}
2686
2687	static const TCGOutOpUnary outop_extrh_i64_i32 = {
2688	.base.static_constraint = C_O1_I1(r, 0),
2689	.out_rr = tgen_extrh_i64_i32,
2690	};
2691
2692	static void tgen_mul(TCGContext *s, TCGType type,
2693	TCGReg a0, TCGReg a1, TCGReg a2)
2694	{
2695	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2696	tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, a0, a2);
2697	}
2698
2699	static void tgen_muli(TCGContext *s, TCGType type,
2700	TCGReg a0, TCGReg a1, tcg_target_long a2)
2701	{
2702	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2703
2704	if (a2 == (int8_t)a2) {
2705	tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, a0, a0);
2706	tcg_out8(s, a2);
2707	} else {
2708	tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, a0, a0);
2709	tcg_out32(s, a2);
2710	}
2711	}
2712
2713	static const TCGOutOpBinary outop_mul = {
2714	.base.static_constraint = C_O1_I2(r, 0, re),
2715	.out_rrr = tgen_mul,
2716	.out_rri = tgen_muli,
2717	};
2718
2719	static void tgen_muls2(TCGContext *s, TCGType type,
2720	TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3)
2721	{
2722	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2723	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, a3);
2724	}
2725
2726	static const TCGOutOpMul2 outop_muls2 = {
2727	.base.static_constraint = C_O2_I2(a, d, a, r),
2728	.out_rrrr = tgen_muls2,
2729	};
2730
2731	static const TCGOutOpBinary outop_mulsh = {
2732	.base.static_constraint = C_NotImplemented,
2733	};
2734
2735	static const TCGOutOpBinary outop_muluh = {
2736	.base.static_constraint = C_NotImplemented,
2737	};
2738
2739	static void tgen_mulu2(TCGContext *s, TCGType type,
2740	TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3)
2741	{
2742	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2743	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, a3);
2744	}
2745
2746	static const TCGOutOpMul2 outop_mulu2 = {
2747	.base.static_constraint = C_O2_I2(a, d, a, r),
2748	.out_rrrr = tgen_mulu2,
2749	};
2750
2751	static const TCGOutOpBinary outop_nand = {
2752	.base.static_constraint = C_NotImplemented,
2753	};
2754
2755	static const TCGOutOpBinary outop_nor = {
2756	.base.static_constraint = C_NotImplemented,
2757	};
2758
2759	static void tgen_or(TCGContext *s, TCGType type,
2760	TCGReg a0, TCGReg a1, TCGReg a2)
2761	{
2762	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2763	tgen_arithr(s, ARITH_OR + rexw, a0, a2);
2764	}
2765
2766	static void tgen_ori(TCGContext *s, TCGType type,
2767	TCGReg a0, TCGReg a1, tcg_target_long a2)
2768	{
2769	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2770	tgen_arithi(s, ARITH_OR + rexw, a0, a2, false);
2771	}
2772
2773	static const TCGOutOpBinary outop_or = {
2774	.base.static_constraint = C_O1_I2(r, 0, re),
2775	.out_rrr = tgen_or,
2776	.out_rri = tgen_ori,
2777	};
2778
2779	static const TCGOutOpBinary outop_orc = {
2780	.base.static_constraint = C_NotImplemented,
2781	};
2782
2783	static const TCGOutOpBinary outop_rems = {
2784	.base.static_constraint = C_NotImplemented,
2785	};
2786
2787	static const TCGOutOpBinary outop_remu = {
2788	.base.static_constraint = C_NotImplemented,
2789	};
2790
2791	static void tgen_rotl(TCGContext *s, TCGType type,
2792	TCGReg a0, TCGReg a1, TCGReg a2)
2793	{
2794	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2795	tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_ROL, a0);
2796	}
2797
2798	static void tgen_rotli(TCGContext *s, TCGType type,
2799	TCGReg a0, TCGReg a1, tcg_target_long a2)
2800	{
2801	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2802	tcg_out_shifti(s, SHIFT_ROL + rexw, a0, a2);
2803	}
2804
2805	static const TCGOutOpBinary outop_rotl = {
2806	.base.static_constraint = C_O1_I2(r, 0, ci),
2807	.out_rrr = tgen_rotl,
2808	.out_rri = tgen_rotli,
2809	};
2810
2811	static void tgen_rotr(TCGContext *s, TCGType type,
2812	TCGReg a0, TCGReg a1, TCGReg a2)
2813	{
2814	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2815	tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_ROR, a0);
2816	}
2817
2818	static void tgen_rotri(TCGContext *s, TCGType type,
2819	TCGReg a0, TCGReg a1, tcg_target_long a2)
2820	{
2821	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2822	tcg_out_shifti(s, SHIFT_ROR + rexw, a0, a2);
2823	}
2824
2825	static const TCGOutOpBinary outop_rotr = {
2826	.base.static_constraint = C_O1_I2(r, 0, ci),
2827	.out_rrr = tgen_rotr,
2828	.out_rri = tgen_rotri,
2829	};
2830
2831	static TCGConstraintSetIndex cset_shift(TCGType type, unsigned flags)
2832	{
2833	return have_bmi2 ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ci);
2834	}
2835
2836	static void tgen_sar(TCGContext *s, TCGType type,
2837	TCGReg a0, TCGReg a1, TCGReg a2)
2838	{
2839	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2840	if (have_bmi2) {
2841	tcg_out_vex_modrm(s, OPC_SARX + rexw, a0, a2, a1);
2842	} else {
2843	tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_SAR, a0);
2844	}
2845	}
2846
2847	static void tgen_sari(TCGContext *s, TCGType type,
2848	TCGReg a0, TCGReg a1, tcg_target_long a2)
2849	{
2850	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2851
2852	tcg_out_mov(s, type, a0, a1);
2853	tcg_out_shifti(s, SHIFT_SAR + rexw, a0, a2);
2854	}
2855
2856	static const TCGOutOpBinary outop_sar = {
2857	.base.static_constraint = C_Dynamic,
2858	.base.dynamic_constraint = cset_shift,
2859	.out_rrr = tgen_sar,
2860	.out_rri = tgen_sari,
2861	};
2862
2863	static void tgen_shl(TCGContext *s, TCGType type,
2864	TCGReg a0, TCGReg a1, TCGReg a2)
2865	{
2866	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2867	if (have_bmi2) {
2868	tcg_out_vex_modrm(s, OPC_SHLX + rexw, a0, a2, a1);
2869	} else {
2870	tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_SHL, a0);
2871	}
2872	}
2873
2874	static void tgen_shli(TCGContext *s, TCGType type,
2875	TCGReg a0, TCGReg a1, tcg_target_long a2)
2876	{
2877	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2878
2879	/* For small constant 3-operand shift, use LEA. */
2880	if (a0 != a1 && a2 >= 1 && a2 <= 3) {
2881	if (a2 == 1) {
2882	/* shl $1,a1,a0 -> lea (a1,a1),a0 */
2883	tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a1, 0, 0);
2884	} else {
2885	/* shl $n,a1,a0 -> lea 0(,a1,n),a0 */
2886	tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, -1, a1, a2, 0);
2887	}
2888	return;
2889	}
2890	tcg_out_mov(s, type, a0, a1);
2891	tcg_out_shifti(s, SHIFT_SHL + rexw, a0, a2);
2892	}
2893
2894	static const TCGOutOpBinary outop_shl = {
2895	.base.static_constraint = C_Dynamic,
2896	.base.dynamic_constraint = cset_shift,
2897	.out_rrr = tgen_shl,
2898	.out_rri = tgen_shli,
2899	};
2900
2901	static void tgen_shr(TCGContext *s, TCGType type,
2902	TCGReg a0, TCGReg a1, TCGReg a2)
2903	{
2904	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2905	if (have_bmi2) {
2906	tcg_out_vex_modrm(s, OPC_SHRX + rexw, a0, a2, a1);
2907	} else {
2908	tcg_out_modrm(s, OPC_SHIFT_cl + rexw, SHIFT_SHR, a0);
2909	}
2910	}
2911
2912	static void tgen_shri(TCGContext *s, TCGType type,
2913	TCGReg a0, TCGReg a1, tcg_target_long a2)
2914	{
2915	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2916
2917	tcg_out_mov(s, type, a0, a1);
2918	tcg_out_shifti(s, SHIFT_SHR + rexw, a0, a2);
2919	}
2920
2921	static const TCGOutOpBinary outop_shr = {
2922	.base.static_constraint = C_Dynamic,
2923	.base.dynamic_constraint = cset_shift,
2924	.out_rrr = tgen_shr,
2925	.out_rri = tgen_shri,
2926	};
2927
2928	static void tgen_sub(TCGContext *s, TCGType type,
2929	TCGReg a0, TCGReg a1, TCGReg a2)
2930	{
2931	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2932	tgen_arithr(s, ARITH_SUB + rexw, a0, a2);
2933	}
2934
2935	static const TCGOutOpSubtract outop_sub = {
2936	.base.static_constraint = C_O1_I2(r, 0, r),
2937	.out_rrr = tgen_sub,
2938	};
2939
2940	static void tgen_subbo_rri(TCGContext *s, TCGType type,
2941	TCGReg a0, TCGReg a1, tcg_target_long a2)
2942	{
2943	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2944	tgen_arithi(s, ARITH_SUB + rexw, a0, a2, 1);
2945	}
2946
2947	static const TCGOutOpAddSubCarry outop_subbo = {
2948	.base.static_constraint = C_O1_I2(r, 0, re),
2949	.out_rrr = tgen_sub,
2950	.out_rri = tgen_subbo_rri,
2951	};
2952
2953	static void tgen_subbio_rrr(TCGContext *s, TCGType type,
2954	TCGReg a0, TCGReg a1, TCGReg a2)
2955	{
2956	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2957	tgen_arithr(s, ARITH_SBB + rexw, a0, a2);
2958	}
2959
2960	static void tgen_subbio_rri(TCGContext *s, TCGType type,
2961	TCGReg a0, TCGReg a1, tcg_target_long a2)
2962	{
2963	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
2964	tgen_arithi(s, ARITH_SBB + rexw, a0, a2, 1);
2965	}
2966
2967	static const TCGOutOpAddSubCarry outop_subbio = {
2968	.base.static_constraint = C_O1_I2(r, 0, re),
2969	.out_rrr = tgen_subbio_rrr,
2970	.out_rri = tgen_subbio_rri,
2971	};
2972
2973	#define outop_subbi outop_subbio
2974
2975	static void tcg_out_set_borrow(TCGContext *s)
2976	{
2977	tcg_out8(s, OPC_STC);
2978	}
2979
2980	static const TCGOutOpBinary outop_smax = {
2981	.base.static_constraint = C_NotImplemented,
2982	};
2983
2984	static const TCGOutOpBinary outop_smin = {
2985	.base.static_constraint = C_NotImplemented,
2986	};
2987
2988	static const TCGOutOpBinary outop_umax = {
2989	.base.static_constraint = C_NotImplemented,
2990	};
2991
2992	static const TCGOutOpBinary outop_umin = {
2993	.base.static_constraint = C_NotImplemented,
2994	};
2995
2996	static void tgen_xor(TCGContext *s, TCGType type,
2997	TCGReg a0, TCGReg a1, TCGReg a2)
2998	{
2999	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
3000	tgen_arithr(s, ARITH_XOR + rexw, a0, a2);
3001	}
3002
3003	static void tgen_xori(TCGContext *s, TCGType type,
3004	TCGReg a0, TCGReg a1, tcg_target_long a2)
3005	{
3006	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
3007	tgen_arithi(s, ARITH_XOR + rexw, a0, a2, false);
3008	}
3009
3010	static const TCGOutOpBinary outop_xor = {
3011	.base.static_constraint = C_O1_I2(r, 0, re),
3012	.out_rrr = tgen_xor,
3013	.out_rri = tgen_xori,
3014	};
3015
3016	static void tgen_bswap16(TCGContext *s, TCGType type,
3017	TCGReg a0, TCGReg a1, unsigned flags)
3018	{
3019	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
3020
3021	if (flags & TCG_BSWAP_OS) {
3022	/* Output must be sign-extended. */
3023	if (rexw) {
3024	tcg_out_bswap64(s, a0);
3025	tcg_out_shifti(s, SHIFT_SAR + rexw, a0, 48);
3026	} else {
3027	tcg_out_bswap32(s, a0);
3028	tcg_out_shifti(s, SHIFT_SAR, a0, 16);
3029	}
3030	} else if ((flags & (TCG_BSWAP_IZ \| TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
3031	/* Output must be zero-extended, but input isn't. */
3032	tcg_out_bswap32(s, a0);
3033	tcg_out_shifti(s, SHIFT_SHR, a0, 16);
3034	} else {
3035	tcg_out_rolw_8(s, a0);
3036	}
3037	}
3038
3039	static const TCGOutOpBswap outop_bswap16 = {
3040	.base.static_constraint = C_O1_I1(r, 0),
3041	.out_rr = tgen_bswap16,
3042	};
3043
3044	static void tgen_bswap32(TCGContext *s, TCGType type,
3045	TCGReg a0, TCGReg a1, unsigned flags)
3046	{
3047	tcg_out_bswap32(s, a0);
3048	if (flags & TCG_BSWAP_OS) {
3049	tcg_out_ext32s(s, a0, a0);
3050	}
3051	}
3052
3053	static const TCGOutOpBswap outop_bswap32 = {
3054	.base.static_constraint = C_O1_I1(r, 0),
3055	.out_rr = tgen_bswap32,
3056	};
3057
3058	static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
3059	{
3060	tcg_out_bswap64(s, a0);
3061	}
3062
3063	static const TCGOutOpUnary outop_bswap64 = {
3064	.base.static_constraint = C_O1_I1(r, 0),
3065	.out_rr = tgen_bswap64,
3066	};
3067
3068	static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
3069	{
3070	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
3071	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, a0);
3072	}
3073
3074	static const TCGOutOpUnary outop_neg = {
3075	.base.static_constraint = C_O1_I1(r, 0),
3076	.out_rr = tgen_neg,
3077	};
3078
3079	static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
3080	{
3081	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
3082	tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
3083	}
3084
3085	static const TCGOutOpUnary outop_not = {
3086	.base.static_constraint = C_O1_I1(r, 0),
3087	.out_rr = tgen_not,
3088	};
3089
3090	static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
3091	TCGReg a2, unsigned ofs, unsigned len)
3092	{
3093	if (ofs == 0 && len == 8) {
3094	tcg_out_modrm(s, OPC_MOVB_EvGv \| P_REXB_R \| P_REXB_RM, a2, a0);
3095	} else if (ofs == 0 && len == 16) {
3096	tcg_out_modrm(s, OPC_MOVL_EvGv \| P_DATA16, a2, a0);
3097	} else {
3098	g_assert_not_reached();
3099	}
3100	}
3101
3102	static void tgen_depositi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
3103	tcg_target_long a2, unsigned ofs, unsigned len)
3104	{
3105	if (ofs == 0 && len == 8) {
3106	tcg_out_opc(s, OPC_MOVB_Ib \| P_REXB_RM \| LOWREGMASK(a0), 0, a0, 0);
3107	tcg_out8(s, a2);
3108	} else if (ofs == 0 && len == 16) {
3109	tcg_out_opc(s, OPC_MOVL_Iv \| P_DATA16 \| LOWREGMASK(a0), 0, a0, 0);
3110	tcg_out16(s, a2);
3111	} else {
3112	g_assert_not_reached();
3113	}
3114	}
3115
3116	static const TCGOutOpDeposit outop_deposit = {
3117	.base.static_constraint = C_O1_I2(q, 0, qi),
3118	.out_rrr = tgen_deposit,
3119	.out_rri = tgen_depositi,
3120	};
3121
3122	static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
3123	unsigned ofs, unsigned len)
3124	{
3125	if (ofs == 0) {
3126	switch (len) {
3127	case 8:
3128	tcg_out_ext8u(s, a0, a1);
3129	return;
3130	case 16:
3131	tcg_out_ext16u(s, a0, a1);
3132	return;
3133	case 32:
3134	tcg_out_ext32u(s, a0, a1);
3135	return;
3136	}
3137	} else if (ofs + len == 32) {
3138	/* This is a 32-bit zero-extending right shift. */
3139	tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
3140	tcg_out_shifti(s, SHIFT_SHR, a0, ofs);
3141	return;
3142	} else if (ofs == 8 && len == 8) {
3143	/*
3144	* On the off-chance that we can use the high-byte registers.
3145	* Otherwise we emit the same ext16 + shift pattern that we
3146	* would have gotten from the normal tcg-op.c expansion.
3147	*/
3148	if (a1 < 4 && a0 < 8) {
3149	tcg_out_modrm(s, OPC_MOVZBL, a0, a1 + 4);
3150	} else {
3151	tcg_out_ext16u(s, a0, a1);
3152	tcg_out_shifti(s, SHIFT_SHR, a0, 8);
3153	}
3154	return;
3155	}
3156	g_assert_not_reached();
3157	}
3158
3159	static const TCGOutOpExtract outop_extract = {
3160	.base.static_constraint = C_O1_I1(r, r),
3161	.out_rr = tgen_extract,
3162	};
3163
3164	static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
3165	unsigned ofs, unsigned len)
3166	{
3167	if (ofs == 0) {
3168	switch (len) {
3169	case 8:
3170	tcg_out_ext8s(s, type, a0, a1);
3171	return;
3172	case 16:
3173	tcg_out_ext16s(s, type, a0, a1);
3174	return;
3175	case 32:
3176	tcg_out_ext32s(s, a0, a1);
3177	return;
3178	}
3179	} else if (ofs == 8 && len == 8) {
3180	if (type == TCG_TYPE_I32 && a1 < 4 && a0 < 8) {
3181	tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4);
3182	} else {
3183	tcg_out_ext16s(s, type, a0, a1);
3184	tgen_sari(s, type, a0, a0, 8);
3185	}
3186	return;
3187	}
3188	g_assert_not_reached();
3189	}
3190
3191	static const TCGOutOpExtract outop_sextract = {
3192	.base.static_constraint = C_O1_I1(r, r),
3193	.out_rr = tgen_sextract,
3194	};
3195
3196	static void tgen_extract2(TCGContext *s, TCGType type, TCGReg a0,
3197	TCGReg a1, TCGReg a2, unsigned shr)
3198	{
3199	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
3200
3201	/* Note that SHRD outputs to the r/m operand. */
3202	tcg_out_modrm(s, OPC_SHRD_Ib + rexw, a2, a0);
3203	tcg_out8(s, shr);
3204	}
3205
3206	static const TCGOutOpExtract2 outop_extract2 = {
3207	.base.static_constraint = C_O1_I2(r, 0, r),
3208	.out_rrr = tgen_extract2,
3209	};
3210
3211	static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest,
3212	TCGReg base, ptrdiff_t offset)
3213	{
3214	tcg_out_modrm_offset(s, OPC_MOVZBL, dest, base, offset);
3215	}
3216
3217	static const TCGOutOpLoad outop_ld8u = {
3218	.base.static_constraint = C_O1_I1(r, r),
3219	.out = tgen_ld8u,
3220	};
3221
3222	static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest,
3223	TCGReg base, ptrdiff_t offset)
3224	{
3225	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
3226	tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, dest, base, offset);
3227	}
3228
3229	static const TCGOutOpLoad outop_ld8s = {
3230	.base.static_constraint = C_O1_I1(r, r),
3231	.out = tgen_ld8s,
3232	};
3233
3234	static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest,
3235	TCGReg base, ptrdiff_t offset)
3236	{
3237	tcg_out_modrm_offset(s, OPC_MOVZWL, dest, base, offset);
3238	}
3239
3240	static const TCGOutOpLoad outop_ld16u = {
3241	.base.static_constraint = C_O1_I1(r, r),
3242	.out = tgen_ld16u,
3243	};
3244
3245	static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest,
3246	TCGReg base, ptrdiff_t offset)
3247	{
3248	int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW;
3249	tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, dest, base, offset);
3250	}
3251
3252	static const TCGOutOpLoad outop_ld16s = {
3253	.base.static_constraint = C_O1_I1(r, r),
3254	.out = tgen_ld16s,
3255	};
3256
3257	static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest,
3258	TCGReg base, ptrdiff_t offset)
3259	{
3260	tcg_out_modrm_offset(s, OPC_MOVL_GvEv, dest, base, offset);
3261	}
3262
3263	static const TCGOutOpLoad outop_ld32u = {
3264	.base.static_constraint = C_O1_I1(r, r),
3265	.out = tgen_ld32u,
3266	};
3267
3268	static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest,
3269	TCGReg base, ptrdiff_t offset)
3270	{
3271	tcg_out_modrm_offset(s, OPC_MOVSLQ, dest, base, offset);
3272	}
3273
3274	static const TCGOutOpLoad outop_ld32s = {
3275	.base.static_constraint = C_O1_I1(r, r),
3276	.out = tgen_ld32s,
3277	};
3278
3279	static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data,
3280	TCGReg base, ptrdiff_t offset)
3281	{
3282	tcg_out_modrm_offset(s, OPC_MOVB_EvGv \| P_REXB_R, data, base, offset);
3283	}
3284
3285	static void tgen_st8_i(TCGContext *s, TCGType type, tcg_target_long data,
3286	TCGReg base, ptrdiff_t offset)
3287	{
3288	tcg_out_modrm_offset(s, OPC_MOVB_EvIz, 0, base, offset);
3289	tcg_out8(s, data);
3290	}
3291
3292	static const TCGOutOpStore outop_st8 = {
3293	.base.static_constraint = C_O0_I2(qi, r),
3294	.out_r = tgen_st8_r,
3295	.out_i = tgen_st8_i,
3296	};
3297
3298	static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data,
3299	TCGReg base, ptrdiff_t offset)
3300	{
3301	tcg_out_modrm_offset(s, OPC_MOVL_EvGv \| P_DATA16, data, base, offset);
3302	}
3303
3304	static void tgen_st16_i(TCGContext *s, TCGType type, tcg_target_long data,
3305	TCGReg base, ptrdiff_t offset)
3306	{
3307	tcg_out_modrm_offset(s, OPC_MOVL_EvIz \| P_DATA16, 0, base, offset);
3308	tcg_out16(s, data);
3309	}
3310
3311	static const TCGOutOpStore outop_st16 = {
3312	.base.static_constraint = C_O0_I2(ri, r),
3313	.out_r = tgen_st16_r,
3314	.out_i = tgen_st16_i,
3315	};
3316
3317	static void tgen_st_i(TCGContext *s, TCGType type, tcg_target_long data,
3318	TCGReg base, ptrdiff_t offset)
3319	{
3320	bool ok = tcg_out_sti(s, type, data, base, offset);
3321	tcg_debug_assert(ok);
3322	}
3323
3324	static const TCGOutOpStore outop_st = {
3325	.base.static_constraint = C_O0_I2(re, r),
3326	.out_r = tcg_out_st,
3327	.out_i = tgen_st_i,
3328	};
3329
3330	static int const umin_insn[4] = {
3331	OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ
3332	};
3333
3334	static int const umax_insn[4] = {
3335	OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_VPMAXUQ
3336	};
3337
3338	static bool tcg_out_cmp_vec_noinv(TCGContext *s, TCGType type, unsigned vece,
3339	TCGReg v0, TCGReg v1, TCGReg v2, TCGCond cond)
3340	{
3341	static int const cmpeq_insn[4] = {
3342	OPC_PCMPEQB, OPC_PCMPEQW, OPC_PCMPEQD, OPC_PCMPEQQ
3343	};
3344	static int const cmpgt_insn[4] = {
3345	OPC_PCMPGTB, OPC_PCMPGTW, OPC_PCMPGTD, OPC_PCMPGTQ
3346	};
3347
3348	enum {
3349	NEED_INV = 1,
3350	NEED_SWAP = 2,
3351	NEED_UMIN = 4,
3352	NEED_UMAX = 8,
3353	INVALID = 16,
3354	};
3355	static const uint8_t cond_fixup[16] = {
3356	[0 ... 15] = INVALID,
3357	[TCG_COND_EQ] = 0,
3358	[TCG_COND_GT] = 0,
3359	[TCG_COND_NE] = NEED_INV,
3360	[TCG_COND_LE] = NEED_INV,
3361	[TCG_COND_LT] = NEED_SWAP,
3362	[TCG_COND_GE] = NEED_SWAP \| NEED_INV,
3363	[TCG_COND_LEU] = NEED_UMIN,
3364	[TCG_COND_GTU] = NEED_UMIN \| NEED_INV,
3365	[TCG_COND_GEU] = NEED_UMAX,
3366	[TCG_COND_LTU] = NEED_UMAX \| NEED_INV,
3367	};
3368	int fixup = cond_fixup[cond];
3369
3370	assert(!(fixup & INVALID));
3371
3372	if (fixup & NEED_INV) {
3373	cond = tcg_invert_cond(cond);
3374	}
3375
3376	if (fixup & NEED_SWAP) {
3377	TCGReg swap = v1;
3378	v1 = v2;
3379	v2 = swap;
3380	cond = tcg_swap_cond(cond);
3381	}
3382
3383	if (fixup & (NEED_UMIN \| NEED_UMAX)) {
3384	int op = (fixup & NEED_UMIN ? umin_insn[vece] : umax_insn[vece]);
3385
3386	/* avx2 does not have 64-bit min/max; adjusted during expand. */
3387	assert(vece <= MO_32);
3388
3389	tcg_out_vex_modrm_type(s, op, TCG_TMP_VEC, v1, v2, type);
3390	v2 = TCG_TMP_VEC;
3391	cond = TCG_COND_EQ;
3392	}
3393
3394	switch (cond) {
3395	case TCG_COND_EQ:
3396	tcg_out_vex_modrm_type(s, cmpeq_insn[vece], v0, v1, v2, type);
3397	break;
3398	case TCG_COND_GT:
3399	tcg_out_vex_modrm_type(s, cmpgt_insn[vece], v0, v1, v2, type);
3400	break;
3401	default:
3402	g_assert_not_reached();
3403	}
3404	return fixup & NEED_INV;
3405	}
3406
3407	static void tcg_out_cmp_vec_k1(TCGContext *s, TCGType type, unsigned vece,
3408	TCGReg v1, TCGReg v2, TCGCond cond)
3409	{
3410	static const int cmpm_insn[2][4] = {
3411	{ OPC_VPCMPB, OPC_VPCMPW, OPC_VPCMPD, OPC_VPCMPQ },
3412	{ OPC_VPCMPUB, OPC_VPCMPUW, OPC_VPCMPUD, OPC_VPCMPUQ }
3413	};
3414	static const int testm_insn[4] = {
3415	OPC_VPTESTMB, OPC_VPTESTMW, OPC_VPTESTMD, OPC_VPTESTMQ
3416	};
3417	static const int testnm_insn[4] = {
3418	OPC_VPTESTNMB, OPC_VPTESTNMW, OPC_VPTESTNMD, OPC_VPTESTNMQ
3419	};
3420
3421	static const int cond_ext[16] = {
3422	[TCG_COND_EQ] = 0,
3423	[TCG_COND_NE] = 4,
3424	[TCG_COND_LT] = 1,
3425	[TCG_COND_LTU] = 1,
3426	[TCG_COND_LE] = 2,
3427	[TCG_COND_LEU] = 2,
3428	[TCG_COND_NEVER] = 3,
3429	[TCG_COND_GE] = 5,
3430	[TCG_COND_GEU] = 5,
3431	[TCG_COND_GT] = 6,
3432	[TCG_COND_GTU] = 6,
3433	[TCG_COND_ALWAYS] = 7,
3434	};
3435
3436	switch (cond) {
3437	case TCG_COND_TSTNE:
3438	tcg_out_vex_modrm_type(s, testm_insn[vece], /* k1 */ 1, v1, v2, type);
3439	break;
3440	case TCG_COND_TSTEQ:
3441	tcg_out_vex_modrm_type(s, testnm_insn[vece], /* k1 */ 1, v1, v2, type);
3442	break;
3443	default:
3444	tcg_out_vex_modrm_type(s, cmpm_insn[is_unsigned_cond(cond)][vece],
3445	/* k1 */ 1, v1, v2, type);
3446	tcg_out8(s, cond_ext[cond]);
3447	break;
3448	}
3449	}
3450
3451	static void tcg_out_k1_to_vec(TCGContext *s, TCGType type,
3452	unsigned vece, TCGReg dest)
3453	{
3454	static const int movm_insn[] = {
3455	OPC_VPMOVM2B, OPC_VPMOVM2W, OPC_VPMOVM2D, OPC_VPMOVM2Q
3456	};
3457	tcg_out_vex_modrm_type(s, movm_insn[vece], dest, 0, /* k1 */ 1, type);
3458	}
3459
3460	static void tcg_out_cmp_vec(TCGContext *s, TCGType type, unsigned vece,
3461	TCGReg v0, TCGReg v1, TCGReg v2, TCGCond cond)
3462	{
3463	/*
3464	* With avx512, we have a complete set of comparisons into mask.
3465	* Unless there's a single insn expansion for the comparision,
3466	* expand via a mask in k1.
3467	*/
3468	if ((vece <= MO_16 ? have_avx512bw : have_avx512dq)
3469	&& cond != TCG_COND_EQ
3470	&& cond != TCG_COND_LT
3471	&& cond != TCG_COND_GT) {
3472	tcg_out_cmp_vec_k1(s, type, vece, v1, v2, cond);
3473	tcg_out_k1_to_vec(s, type, vece, v0);
3474	return;
3475	}
3476
3477	if (tcg_out_cmp_vec_noinv(s, type, vece, v0, v1, v2, cond)) {
3478	tcg_out_dupi_vec(s, type, vece, TCG_TMP_VEC, -1);
3479	tcg_out_vex_modrm_type(s, OPC_PXOR, v0, v0, TCG_TMP_VEC, type);
3480	}
3481	}
3482
3483	static void tcg_out_cmpsel_vec_k1(TCGContext *s, TCGType type, unsigned vece,
3484	TCGReg v0, TCGReg c1, TCGReg c2,
3485	TCGReg v3, TCGReg v4, TCGCond cond)
3486	{
3487	static const int vpblendm_insn[] = {
3488	OPC_VPBLENDMB, OPC_VPBLENDMW, OPC_VPBLENDMD, OPC_VPBLENDMQ
3489	};
3490	bool z = false;
3491
3492	/* Swap to place constant in V4 to take advantage of zero-masking. */
3493	if (!v3) {
3494	z = true;
3495	v3 = v4;
3496	cond = tcg_invert_cond(cond);
3497	}
3498
3499	tcg_out_cmp_vec_k1(s, type, vece, c1, c2, cond);
3500	tcg_out_evex_modrm_type(s, vpblendm_insn[vece], v0, v4, v3,
3501	/* k1 */1, z, type);
3502	}
3503
3504	static void tcg_out_cmpsel_vec(TCGContext *s, TCGType type, unsigned vece,
3505	TCGReg v0, TCGReg c1, TCGReg c2,
3506	TCGReg v3, TCGReg v4, TCGCond cond)
3507	{
3508	bool inv;
3509
3510	if (vece <= MO_16 ? have_avx512bw : have_avx512vl) {
3511	tcg_out_cmpsel_vec_k1(s, type, vece, v0, c1, c2, v3, v4, cond);
3512	return;
3513	}
3514
3515	inv = tcg_out_cmp_vec_noinv(s, type, vece, TCG_TMP_VEC, c1, c2, cond);
3516
3517	/*
3518	* Since XMM0 is 16, the only way we get 0 into V3
3519	* is via the constant zero constraint.
3520	*/
3521	if (!v3) {
3522	if (inv) {
3523	tcg_out_vex_modrm_type(s, OPC_PAND, v0, TCG_TMP_VEC, v4, type);
3524	} else {
3525	tcg_out_vex_modrm_type(s, OPC_PANDN, v0, TCG_TMP_VEC, v4, type);
3526	}
3527	} else {
3528	if (inv) {
3529	TCGReg swap = v3;
3530	v3 = v4;
3531	v4 = swap;
3532	}
3533	tcg_out_vex_modrm_type(s, OPC_VPBLENDVB, v0, v4, v3, type);
3534	tcg_out8(s, (TCG_TMP_VEC - TCG_REG_XMM0) << 4);
3535	}
3536	}
3537
3538	static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
3539	unsigned vecl, unsigned vece,
3540	const TCGArg args[TCG_MAX_OP_ARGS],
3541	const int const_args[TCG_MAX_OP_ARGS])
3542	{
3543	static int const add_insn[4] = {
3544	OPC_PADDB, OPC_PADDW, OPC_PADDD, OPC_PADDQ
3545	};
3546	static int const ssadd_insn[4] = {
3547	OPC_PADDSB, OPC_PADDSW, OPC_UD2, OPC_UD2
3548	};
3549	static int const usadd_insn[4] = {
3550	OPC_PADDUB, OPC_PADDUW, OPC_UD2, OPC_UD2
3551	};
3552	static int const sub_insn[4] = {
3553	OPC_PSUBB, OPC_PSUBW, OPC_PSUBD, OPC_PSUBQ
3554	};
3555	static int const sssub_insn[4] = {
3556	OPC_PSUBSB, OPC_PSUBSW, OPC_UD2, OPC_UD2
3557	};
3558	static int const ussub_insn[4] = {
3559	OPC_PSUBUB, OPC_PSUBUW, OPC_UD2, OPC_UD2
3560	};
3561	static int const mul_insn[4] = {
3562	OPC_UD2, OPC_PMULLW, OPC_PMULLD, OPC_VPMULLQ
3563	};
3564	static int const shift_imm_insn[4] = {
3565	OPC_UD2, OPC_PSHIFTW_Ib, OPC_PSHIFTD_Ib, OPC_PSHIFTQ_Ib
3566	};
3567	static int const punpckl_insn[4] = {
3568	OPC_PUNPCKLBW, OPC_PUNPCKLWD, OPC_PUNPCKLDQ, OPC_PUNPCKLQDQ
3569	};
3570	static int const punpckh_insn[4] = {
3571	OPC_PUNPCKHBW, OPC_PUNPCKHWD, OPC_PUNPCKHDQ, OPC_PUNPCKHQDQ
3572	};
3573	static int const packss_insn[4] = {
3574	OPC_PACKSSWB, OPC_PACKSSDW, OPC_UD2, OPC_UD2
3575	};
3576	static int const packus_insn[4] = {
3577	OPC_PACKUSWB, OPC_PACKUSDW, OPC_UD2, OPC_UD2
3578	};
3579	static int const smin_insn[4] = {
3580	OPC_PMINSB, OPC_PMINSW, OPC_PMINSD, OPC_VPMINSQ
3581	};
3582	static int const smax_insn[4] = {
3583	OPC_PMAXSB, OPC_PMAXSW, OPC_PMAXSD, OPC_VPMAXSQ
3584	};
3585	static int const rotlv_insn[4] = {
3586	OPC_UD2, OPC_UD2, OPC_VPROLVD, OPC_VPROLVQ
3587	};
3588	static int const rotrv_insn[4] = {
3589	OPC_UD2, OPC_UD2, OPC_VPRORVD, OPC_VPRORVQ
3590	};
3591	static int const shlv_insn[4] = {
3592	OPC_UD2, OPC_VPSLLVW, OPC_VPSLLVD, OPC_VPSLLVQ
3593	};
3594	static int const shrv_insn[4] = {
3595	OPC_UD2, OPC_VPSRLVW, OPC_VPSRLVD, OPC_VPSRLVQ
3596	};
3597	static int const sarv_insn[4] = {
3598	OPC_UD2, OPC_VPSRAVW, OPC_VPSRAVD, OPC_VPSRAVQ
3599	};
3600	static int const shls_insn[4] = {
3601	OPC_UD2, OPC_PSLLW, OPC_PSLLD, OPC_PSLLQ
3602	};
3603	static int const shrs_insn[4] = {
3604	OPC_UD2, OPC_PSRLW, OPC_PSRLD, OPC_PSRLQ
3605	};
3606	static int const sars_insn[4] = {
3607	OPC_UD2, OPC_PSRAW, OPC_PSRAD, OPC_VPSRAQ
3608	};
3609	static int const vpshldi_insn[4] = {
3610	OPC_UD2, OPC_VPSHLDW, OPC_VPSHLDD, OPC_VPSHLDQ
3611	};
3612	static int const vpshldv_insn[4] = {
3613	OPC_UD2, OPC_VPSHLDVW, OPC_VPSHLDVD, OPC_VPSHLDVQ
3614	};
3615	static int const vpshrdv_insn[4] = {
3616	OPC_UD2, OPC_VPSHRDVW, OPC_VPSHRDVD, OPC_VPSHRDVQ
3617	};
3618	static int const abs_insn[4] = {
3619	OPC_PABSB, OPC_PABSW, OPC_PABSD, OPC_VPABSQ
3620	};
3621
3622	TCGType type = vecl + TCG_TYPE_V64;
3623	int insn, sub;
3624	TCGArg a0, a1, a2, a3;
3625
3626	a0 = args[0];
3627	a1 = args[1];
3628	a2 = args[2];
3629
3630	switch (opc) {
3631	case INDEX_op_add_vec:
3632	insn = add_insn[vece];
3633	goto gen_simd;
3634	case INDEX_op_ssadd_vec:
3635	insn = ssadd_insn[vece];
3636	goto gen_simd;
3637	case INDEX_op_usadd_vec:
3638	insn = usadd_insn[vece];
3639	goto gen_simd;
3640	case INDEX_op_sub_vec:
3641	insn = sub_insn[vece];
3642	goto gen_simd;
3643	case INDEX_op_sssub_vec:
3644	insn = sssub_insn[vece];
3645	goto gen_simd;
3646	case INDEX_op_ussub_vec:
3647	insn = ussub_insn[vece];
3648	goto gen_simd;
3649	case INDEX_op_mul_vec:
3650	insn = mul_insn[vece];
3651	goto gen_simd;
3652	case INDEX_op_and_vec:
3653	insn = OPC_PAND;
3654	goto gen_simd;
3655	case INDEX_op_or_vec:
3656	insn = OPC_POR;
3657	goto gen_simd;
3658	case INDEX_op_xor_vec:
3659	insn = OPC_PXOR;
3660	goto gen_simd;
3661	case INDEX_op_smin_vec:
3662	insn = smin_insn[vece];
3663	goto gen_simd;
3664	case INDEX_op_umin_vec:
3665	insn = umin_insn[vece];
3666	goto gen_simd;
3667	case INDEX_op_smax_vec:
3668	insn = smax_insn[vece];
3669	goto gen_simd;
3670	case INDEX_op_umax_vec:
3671	insn = umax_insn[vece];
3672	goto gen_simd;
3673	case INDEX_op_shlv_vec:
3674	insn = shlv_insn[vece];
3675	goto gen_simd;
3676	case INDEX_op_shrv_vec:
3677	insn = shrv_insn[vece];
3678	goto gen_simd;
3679	case INDEX_op_sarv_vec:
3680	insn = sarv_insn[vece];
3681	goto gen_simd;
3682	case INDEX_op_rotlv_vec:
3683	insn = rotlv_insn[vece];
3684	goto gen_simd;
3685	case INDEX_op_rotrv_vec:
3686	insn = rotrv_insn[vece];
3687	goto gen_simd;
3688	case INDEX_op_shls_vec:
3689	insn = shls_insn[vece];
3690	goto gen_simd;
3691	case INDEX_op_shrs_vec:
3692	insn = shrs_insn[vece];
3693	goto gen_simd;
3694	case INDEX_op_sars_vec:
3695	insn = sars_insn[vece];
3696	goto gen_simd;
3697	case INDEX_op_x86_punpckl_vec:
3698	insn = punpckl_insn[vece];
3699	goto gen_simd;
3700	case INDEX_op_x86_punpckh_vec:
3701	insn = punpckh_insn[vece];
3702	goto gen_simd;
3703	case INDEX_op_x86_packss_vec:
3704	insn = packss_insn[vece];
3705	goto gen_simd;
3706	case INDEX_op_x86_packus_vec:
3707	insn = packus_insn[vece];
3708	goto gen_simd;
3709	case INDEX_op_x86_vpshldv_vec:
3710	insn = vpshldv_insn[vece];
3711	a1 = a2;
3712	a2 = args[3];
3713	goto gen_simd;
3714	case INDEX_op_x86_vpshrdv_vec:
3715	insn = vpshrdv_insn[vece];
3716	a1 = a2;
3717	a2 = args[3];
3718	goto gen_simd;
3719	case INDEX_op_abs_vec:
3720	insn = abs_insn[vece];
3721	a2 = a1;
3722	a1 = 0;
3723	goto gen_simd;
3724	gen_simd:
3725	tcg_debug_assert(insn != OPC_UD2);
3726	tcg_out_vex_modrm_type(s, insn, a0, a1, a2, type);
3727	break;
3728
3729	case INDEX_op_cmp_vec:
3730	tcg_out_cmp_vec(s, type, vece, a0, a1, a2, args[3]);
3731	break;
3732
3733	case INDEX_op_cmpsel_vec:
3734	tcg_out_cmpsel_vec(s, type, vece, a0, a1, a2,
3735	args[3], args[4], args[5]);
3736	break;
3737
3738	case INDEX_op_andc_vec:
3739	insn = OPC_PANDN;
3740	tcg_out_vex_modrm_type(s, insn, a0, a2, a1, type);
3741	break;
3742
3743	case INDEX_op_shli_vec:
3744	insn = shift_imm_insn[vece];
3745	sub = 6;
3746	goto gen_shift;
3747	case INDEX_op_shri_vec:
3748	insn = shift_imm_insn[vece];
3749	sub = 2;
3750	goto gen_shift;
3751	case INDEX_op_sari_vec:
3752	if (vece == MO_64) {
3753	insn = OPC_PSHIFTD_Ib \| P_VEXW \| P_EVEX;
3754	} else {
3755	insn = shift_imm_insn[vece];
3756	}
3757	sub = 4;
3758	goto gen_shift;
3759	case INDEX_op_rotli_vec:
3760	insn = OPC_PSHIFTD_Ib \| P_EVEX; /* VPROL[DQ] */
3761	if (vece == MO_64) {
3762	insn \|= P_VEXW;
3763	}
3764	sub = 1;
3765	goto gen_shift;
3766	gen_shift:
3767	tcg_debug_assert(vece != MO_8);
3768	tcg_out_vex_modrm_type(s, insn, sub, a0, a1, type);
3769	tcg_out8(s, a2);
3770	break;
3771
3772	case INDEX_op_ld_vec:
3773	tcg_out_ld(s, type, a0, a1, a2);
3774	break;
3775	case INDEX_op_st_vec:
3776	tcg_out_st(s, type, a0, a1, a2);
3777	break;
3778	case INDEX_op_dupm_vec:
3779	tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
3780	break;
3781
3782	case INDEX_op_x86_shufps_vec:
3783	insn = OPC_SHUFPS;
3784	sub = args[3];
3785	goto gen_simd_imm8;
3786	case INDEX_op_x86_blend_vec:
3787	if (vece == MO_16) {
3788	insn = OPC_PBLENDW;
3789	} else if (vece == MO_32) {
3790	insn = (have_avx2 ? OPC_VPBLENDD : OPC_BLENDPS);
3791	} else {
3792	g_assert_not_reached();
3793	}
3794	sub = args[3];
3795	goto gen_simd_imm8;
3796	case INDEX_op_x86_vperm2i128_vec:
3797	insn = OPC_VPERM2I128;
3798	sub = args[3];
3799	goto gen_simd_imm8;
3800	case INDEX_op_x86_vpshldi_vec:
3801	insn = vpshldi_insn[vece];
3802	sub = args[3];
3803	goto gen_simd_imm8;
3804	case INDEX_op_x86_vgf2p8affineqb_vec:
3805	insn = OPC_VGF2P8AFFINEQB;
3806	sub = args[3];
3807	goto gen_simd_imm8;
3808
3809	case INDEX_op_not_vec:
3810	insn = OPC_VPTERNLOGQ;
3811	a2 = a1;
3812	sub = 0x33; /* !B */
3813	goto gen_simd_imm8;
3814	case INDEX_op_nor_vec:
3815	insn = OPC_VPTERNLOGQ;
3816	sub = 0x11; /* norCB */
3817	goto gen_simd_imm8;
3818	case INDEX_op_nand_vec:
3819	insn = OPC_VPTERNLOGQ;
3820	sub = 0x77; /* nandCB */
3821	goto gen_simd_imm8;
3822	case INDEX_op_eqv_vec:
3823	insn = OPC_VPTERNLOGQ;
3824	sub = 0x99; /* xnorCB */
3825	goto gen_simd_imm8;
3826	case INDEX_op_orc_vec:
3827	insn = OPC_VPTERNLOGQ;
3828	sub = 0xdd; /* orB!C */
3829	goto gen_simd_imm8;
3830
3831	case INDEX_op_bitsel_vec:
3832	insn = OPC_VPTERNLOGQ;
3833	a3 = args[3];
3834	if (a0 == a1) {
3835	a1 = a2;
3836	a2 = a3;
3837	sub = 0xca; /* A?B:C */
3838	} else if (a0 == a2) {
3839	a2 = a3;
3840	sub = 0xe2; /* B?A:C */
3841	} else {
3842	tcg_out_mov(s, type, a0, a3);
3843	sub = 0xb8; /* B?C:A */
3844	}
3845	goto gen_simd_imm8;
3846
3847	gen_simd_imm8:
3848	tcg_debug_assert(insn != OPC_UD2);
3849	tcg_out_vex_modrm_type(s, insn, a0, a1, a2, type);
3850	tcg_out8(s, sub);
3851	break;
3852
3853	case INDEX_op_x86_psrldq_vec:
3854	tcg_out_vex_modrm(s, OPC_GRP14, 3, a0, a1);
3855	tcg_out8(s, a2);
3856	break;
3857
3858	case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
3859	case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
3860	default:
3861	g_assert_not_reached();
3862	}
3863	}
3864
3865	static TCGConstraintSetIndex
3866	tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags)
3867	{
3868	switch (op) {
3869	case INDEX_op_ld_vec:
3870	case INDEX_op_dupm_vec:
3871	return C_O1_I1(x, r);
3872
3873	case INDEX_op_st_vec:
3874	return C_O0_I2(x, r);
3875
3876	case INDEX_op_add_vec:
3877	case INDEX_op_sub_vec:
3878	case INDEX_op_mul_vec:
3879	case INDEX_op_and_vec:
3880	case INDEX_op_or_vec:
3881	case INDEX_op_xor_vec:
3882	case INDEX_op_andc_vec:
3883	case INDEX_op_orc_vec:
3884	case INDEX_op_nand_vec:
3885	case INDEX_op_nor_vec:
3886	case INDEX_op_eqv_vec:
3887	case INDEX_op_ssadd_vec:
3888	case INDEX_op_usadd_vec:
3889	case INDEX_op_sssub_vec:
3890	case INDEX_op_ussub_vec:
3891	case INDEX_op_smin_vec:
3892	case INDEX_op_umin_vec:
3893	case INDEX_op_smax_vec:
3894	case INDEX_op_umax_vec:
3895	case INDEX_op_shlv_vec:
3896	case INDEX_op_shrv_vec:
3897	case INDEX_op_sarv_vec:
3898	case INDEX_op_rotlv_vec:
3899	case INDEX_op_rotrv_vec:
3900	case INDEX_op_shls_vec:
3901	case INDEX_op_shrs_vec:
3902	case INDEX_op_sars_vec:
3903	case INDEX_op_cmp_vec:
3904	case INDEX_op_x86_shufps_vec:
3905	case INDEX_op_x86_blend_vec:
3906	case INDEX_op_x86_packss_vec:
3907	case INDEX_op_x86_packus_vec:
3908	case INDEX_op_x86_vperm2i128_vec:
3909	case INDEX_op_x86_punpckl_vec:
3910	case INDEX_op_x86_punpckh_vec:
3911	case INDEX_op_x86_vpshldi_vec:
3912	case INDEX_op_x86_vgf2p8affineqb_vec:
3913	return C_O1_I2(x, x, x);
3914
3915	case INDEX_op_abs_vec:
3916	case INDEX_op_dup_vec:
3917	case INDEX_op_not_vec:
3918	case INDEX_op_shli_vec:
3919	case INDEX_op_shri_vec:
3920	case INDEX_op_sari_vec:
3921	case INDEX_op_rotli_vec:
3922	case INDEX_op_x86_psrldq_vec:
3923	return C_O1_I1(x, x);
3924
3925	case INDEX_op_x86_vpshldv_vec:
3926	case INDEX_op_x86_vpshrdv_vec:
3927	return C_O1_I3(x, 0, x, x);
3928
3929	case INDEX_op_bitsel_vec:
3930	return C_O1_I3(x, x, x, x);
3931	case INDEX_op_cmpsel_vec:
3932	return C_O1_I4(x, x, x, xO, x);
3933
3934	default:
3935	return C_NotImplemented;
3936	}
3937	}
3938
3939	int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
3940	{
3941	switch (opc) {
3942	case INDEX_op_add_vec:
3943	case INDEX_op_sub_vec:
3944	case INDEX_op_and_vec:
3945	case INDEX_op_or_vec:
3946	case INDEX_op_xor_vec:
3947	case INDEX_op_andc_vec:
3948	case INDEX_op_orc_vec:
3949	case INDEX_op_nand_vec:
3950	case INDEX_op_nor_vec:
3951	case INDEX_op_eqv_vec:
3952	case INDEX_op_not_vec:
3953	case INDEX_op_bitsel_vec:
3954	return 1;
3955	case INDEX_op_cmp_vec:
3956	case INDEX_op_cmpsel_vec:
3957	return -1;
3958
3959	case INDEX_op_rotli_vec:
3960	return have_avx512vl && vece >= MO_32 ? 1 : -1;
3961
3962	case INDEX_op_shli_vec:
3963	case INDEX_op_shri_vec:
3964	/* We must expand the operation for MO_8. */
3965	return vece == MO_8 ? -1 : 1;
3966
3967	case INDEX_op_sari_vec:
3968	switch (vece) {
3969	case MO_8:
3970	return -1;
3971	case MO_16:
3972	case MO_32:
3973	return 1;
3974	case MO_64:
3975	if (have_avx512vl) {
3976	return 1;
3977	}
3978	/*
3979	* We can emulate this for MO_64, but it does not pay off
3980	* unless we're producing at least 4 values.
3981	*/
3982	return type >= TCG_TYPE_V256 ? -1 : 0;
3983	}
3984	return 0;
3985
3986	case INDEX_op_shls_vec:
3987	case INDEX_op_shrs_vec:
3988	return vece >= MO_16;
3989	case INDEX_op_sars_vec:
3990	switch (vece) {
3991	case MO_16:
3992	case MO_32:
3993	return 1;
3994	case MO_64:
3995	return have_avx512vl;
3996	}
3997	return 0;
3998	case INDEX_op_rotls_vec:
3999	return vece >= MO_16 ? -1 : 0;
4000
4001	case INDEX_op_shlv_vec:
4002	case INDEX_op_shrv_vec:
4003	switch (vece) {
4004	case MO_16:
4005	return have_avx512bw;
4006	case MO_32:
4007	case MO_64:
4008	return have_avx2;
4009	}
4010	return 0;
4011	case INDEX_op_sarv_vec:
4012	switch (vece) {
4013	case MO_16:
4014	return have_avx512bw;
4015	case MO_32:
4016	return have_avx2;
4017	case MO_64:
4018	return have_avx512vl;
4019	}
4020	return 0;
4021	case INDEX_op_rotlv_vec:
4022	case INDEX_op_rotrv_vec:
4023	switch (vece) {
4024	case MO_16:
4025	return have_avx512vbmi2 ? -1 : 0;
4026	case MO_32:
4027	case MO_64:
4028	return have_avx512vl ? 1 : have_avx2 ? -1 : 0;
4029	}
4030	return 0;
4031
4032	case INDEX_op_mul_vec:
4033	switch (vece) {
4034	case MO_8:
4035	return -1;
4036	case MO_64:
4037	return have_avx512dq;
4038	}
4039	return 1;
4040
4041	case INDEX_op_ssadd_vec:
4042	case INDEX_op_usadd_vec:
4043	case INDEX_op_sssub_vec:
4044	case INDEX_op_ussub_vec:
4045	return vece <= MO_16;
4046	case INDEX_op_smin_vec:
4047	case INDEX_op_smax_vec:
4048	case INDEX_op_umin_vec:
4049	case INDEX_op_umax_vec:
4050	case INDEX_op_abs_vec:
4051	return vece <= MO_32 \|\| have_avx512vl;
4052
4053	default:
4054	return 0;
4055	}
4056	}
4057
4058	static void gen_vgf2p8affineqb0(TCGType type, TCGv_vec v0,
4059	TCGv_vec v1, uint64_t matrix)
4060	{
4061	vec_gen_4(INDEX_op_x86_vgf2p8affineqb_vec, type, MO_8,
4062	tcgv_vec_arg(v0), tcgv_vec_arg(v1),
4063	tcgv_vec_arg(tcg_constant_vec(type, MO_64, matrix)), 0);
4064	}
4065
4066	static void expand_vec_shi(TCGType type, unsigned vece, bool right,
4067	TCGv_vec v0, TCGv_vec v1, TCGArg imm)
4068	{
4069	static const uint64_t gf2_shi[2][8] = {
4070	/* left shift */
4071	{ 0,
4072	0x0001020408102040ull,
4073	0x0000010204081020ull,
4074	0x0000000102040810ull,
4075	0x0000000001020408ull,
4076	0x0000000000010204ull,
4077	0x0000000000000102ull,
4078	0x0000000000000001ull },
4079	/* right shift */
4080	{ 0,
4081	0x0204081020408000ull,
4082	0x0408102040800000ull,
4083	0x0810204080000000ull,
4084	0x1020408000000000ull,
4085	0x2040800000000000ull,
4086	0x4080000000000000ull,
4087	0x8000000000000000ull }
4088	};
4089	uint8_t mask;
4090
4091	tcg_debug_assert(vece == MO_8);
4092
4093	if (cpuinfo & CPUINFO_GFNI) {
4094	gen_vgf2p8affineqb0(type, v0, v1, gf2_shi[right][imm]);
4095	return;
4096	}
4097
4098	if (right) {
4099	mask = 0xff >> imm;
4100	tcg_gen_shri_vec(MO_16, v0, v1, imm);
4101	} else {
4102	mask = 0xff << imm;
4103	tcg_gen_shli_vec(MO_16, v0, v1, imm);
4104	}
4105	tcg_gen_and_vec(MO_8, v0, v0, tcg_constant_vec(type, MO_8, mask));
4106	}
4107
4108	static void expand_vec_sari(TCGType type, unsigned vece,
4109	TCGv_vec v0, TCGv_vec v1, TCGArg imm)
4110	{
4111	static const uint64_t gf2_sar[8] = {
4112	0,
4113	0x0204081020408080ull,
4114	0x0408102040808080ull,
4115	0x0810204080808080ull,
4116	0x1020408080808080ull,
4117	0x2040808080808080ull,
4118	0x4080808080808080ull,
4119	0x8080808080808080ull,
4120	};
4121	TCGv_vec t1, t2;
4122
4123	if (imm >= (8 << vece) - 1) {
4124	tcg_gen_cmp_vec(TCG_COND_LT, vece, v0, v1,
4125	tcg_constant_vec(type, MO_64, 0));
4126	return;
4127	}
4128
4129	switch (vece) {
4130	case MO_8:
4131	if (cpuinfo & CPUINFO_GFNI) {
4132	gen_vgf2p8affineqb0(type, v0, v1, gf2_sar[imm]);
4133	break;
4134	}
4135
4136	/* Unpack to 16-bit, shift, and repack. */
4137	t1 = tcg_temp_new_vec(type);
4138	t2 = tcg_temp_new_vec(type);
4139	vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
4140	tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
4141	vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
4142	tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
4143	tcg_gen_sari_vec(MO_16, t1, t1, imm + 8);
4144	tcg_gen_sari_vec(MO_16, t2, t2, imm + 8);
4145	vec_gen_3(INDEX_op_x86_packss_vec, type, MO_8,
4146	tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2));
4147	tcg_temp_free_vec(t1);
4148	tcg_temp_free_vec(t2);
4149	break;
4150
4151	case MO_64:
4152	t1 = tcg_temp_new_vec(type);
4153	if (imm <= 32) {
4154	/*
4155	* We can emulate a small sign extend by performing an arithmetic
4156	* 32-bit shift and overwriting the high half of a 64-bit logical
4157	* shift. Note that the ISA says shift of 32 is valid, but TCG
4158	* does not, so we have to bound the smaller shift -- we get the
4159	* same result in the high half either way.
4160	*/
4161	tcg_gen_sari_vec(MO_32, t1, v1, MIN(imm, 31));
4162	tcg_gen_shri_vec(MO_64, v0, v1, imm);
4163	vec_gen_4(INDEX_op_x86_blend_vec, type, MO_32,
4164	tcgv_vec_arg(v0), tcgv_vec_arg(v0),
4165	tcgv_vec_arg(t1), 0xaa);
4166	} else {
4167	/* Otherwise we will need to use a compare vs 0 to produce
4168	* the sign-extend, shift and merge.
4169	*/
4170	tcg_gen_cmp_vec(TCG_COND_LT, MO_64, t1, v1,
4171	tcg_constant_vec(type, MO_64, 0));
4172	tcg_gen_shri_vec(MO_64, v0, v1, imm);
4173	tcg_gen_shli_vec(MO_64, t1, t1, 64 - imm);
4174	tcg_gen_or_vec(MO_64, v0, v0, t1);
4175	}
4176	tcg_temp_free_vec(t1);
4177	break;
4178
4179	default:
4180	g_assert_not_reached();
4181	}
4182	}
4183
4184	static void expand_vec_rotli(TCGType type, unsigned vece,
4185	TCGv_vec v0, TCGv_vec v1, TCGArg imm)
4186	{
4187	static const uint64_t gf2_rol[8] = {
4188	0,
4189	0x8001020408102040ull,
4190	0x4080010204081020ull,
4191	0x2040800102040810ull,
4192	0x1020408001020408ull,
4193	0x0810204080010204ull,
4194	0x0408102040800102ull,
4195	0x0204081020408001ull,
4196	};
4197	TCGv_vec t;
4198
4199	if (vece == MO_8) {
4200	if (cpuinfo & CPUINFO_GFNI) {
4201	gen_vgf2p8affineqb0(type, v0, v1, gf2_rol[imm]);
4202	return;
4203	}
4204	} else {
4205	if (have_avx512vbmi2) {
4206	vec_gen_4(INDEX_op_x86_vpshldi_vec, type, vece,
4207	tcgv_vec_arg(v0), tcgv_vec_arg(v1),
4208	tcgv_vec_arg(v1), imm);
4209	return;
4210	}
4211	}
4212
4213	t = tcg_temp_new_vec(type);
4214	tcg_gen_shli_vec(vece, t, v1, imm);
4215	tcg_gen_shri_vec(vece, v0, v1, (8 << vece) - imm);
4216	tcg_gen_or_vec(vece, v0, v0, t);
4217	tcg_temp_free_vec(t);
4218	}
4219
4220	static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0,
4221	TCGv_vec v1, TCGv_vec sh, bool right)
4222	{
4223	TCGv_vec t;
4224
4225	if (have_avx512vbmi2) {
4226	vec_gen_4(right ? INDEX_op_x86_vpshrdv_vec : INDEX_op_x86_vpshldv_vec,
4227	type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v1),
4228	tcgv_vec_arg(v1), tcgv_vec_arg(sh));
4229	return;
4230	}
4231
4232	t = tcg_temp_new_vec(type);
4233	tcg_gen_dupi_vec(vece, t, 8 << vece);
4234	tcg_gen_sub_vec(vece, t, t, sh);
4235	if (right) {
4236	tcg_gen_shlv_vec(vece, t, v1, t);
4237	tcg_gen_shrv_vec(vece, v0, v1, sh);
4238	} else {
4239	tcg_gen_shrv_vec(vece, t, v1, t);
4240	tcg_gen_shlv_vec(vece, v0, v1, sh);
4241	}
4242	tcg_gen_or_vec(vece, v0, v0, t);
4243	tcg_temp_free_vec(t);
4244	}
4245
4246	static void expand_vec_rotls(TCGType type, unsigned vece,
4247	TCGv_vec v0, TCGv_vec v1, TCGv_i32 lsh)
4248	{
4249	TCGv_vec t = tcg_temp_new_vec(type);
4250
4251	tcg_debug_assert(vece != MO_8);
4252
4253	if (vece >= MO_32 ? have_avx512vl : have_avx512vbmi2) {
4254	tcg_gen_dup_i32_vec(vece, t, lsh);
4255	if (vece >= MO_32) {
4256	tcg_gen_rotlv_vec(vece, v0, v1, t);
4257	} else {
4258	expand_vec_rotv(type, vece, v0, v1, t, false);
4259	}
4260	} else {
4261	TCGv_i32 rsh = tcg_temp_new_i32();
4262
4263	tcg_gen_neg_i32(rsh, lsh);
4264	tcg_gen_andi_i32(rsh, rsh, (8 << vece) - 1);
4265	tcg_gen_shls_vec(vece, t, v1, lsh);
4266	tcg_gen_shrs_vec(vece, v0, v1, rsh);
4267	tcg_gen_or_vec(vece, v0, v0, t);
4268
4269	tcg_temp_free_i32(rsh);
4270	}
4271
4272	tcg_temp_free_vec(t);
4273	}
4274
4275	static void expand_vec_mul(TCGType type, unsigned vece,
4276	TCGv_vec v0, TCGv_vec v1, TCGv_vec v2)
4277	{
4278	TCGv_vec t1, t2, t3, t4, zero;
4279
4280	tcg_debug_assert(vece == MO_8);
4281
4282	/*
4283	* Unpack v1 bytes to words, 0 \| x.
4284	* Unpack v2 bytes to words, y \| 0.
4285	* This leaves the 8-bit result, x * y, with 8 bits of right padding.
4286	* Shift logical right by 8 bits to clear the high 8 bytes before
4287	* using an unsigned saturated pack.
4288	*
4289	* The difference between the V64, V128 and V256 cases is merely how
4290	* we distribute the expansion between temporaries.
4291	*/
4292	switch (type) {
4293	case TCG_TYPE_V64:
4294	t1 = tcg_temp_new_vec(TCG_TYPE_V128);
4295	t2 = tcg_temp_new_vec(TCG_TYPE_V128);
4296	zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0);
4297	vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8,
4298	tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
4299	vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8,
4300	tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
4301	tcg_gen_mul_vec(MO_16, t1, t1, t2);
4302	tcg_gen_shri_vec(MO_16, t1, t1, 8);
4303	vec_gen_3(INDEX_op_x86_packus_vec, TCG_TYPE_V128, MO_8,
4304	tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t1));
4305	tcg_temp_free_vec(t1);
4306	tcg_temp_free_vec(t2);
4307	break;
4308
4309	case TCG_TYPE_V128:
4310	case TCG_TYPE_V256:
4311	t1 = tcg_temp_new_vec(type);
4312	t2 = tcg_temp_new_vec(type);
4313	t3 = tcg_temp_new_vec(type);
4314	t4 = tcg_temp_new_vec(type);
4315	zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0);
4316	vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
4317	tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
4318	vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
4319	tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
4320	vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
4321	tcgv_vec_arg(t3), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
4322	vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
4323	tcgv_vec_arg(t4), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
4324	tcg_gen_mul_vec(MO_16, t1, t1, t2);
4325	tcg_gen_mul_vec(MO_16, t3, t3, t4);
4326	tcg_gen_shri_vec(MO_16, t1, t1, 8);
4327	tcg_gen_shri_vec(MO_16, t3, t3, 8);
4328	vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8,
4329	tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t3));
4330	tcg_temp_free_vec(t1);
4331	tcg_temp_free_vec(t2);
4332	tcg_temp_free_vec(t3);
4333	tcg_temp_free_vec(t4);
4334	break;
4335
4336	default:
4337	g_assert_not_reached();
4338	}
4339	}
4340
4341	static TCGCond expand_vec_cond(TCGType type, unsigned vece,
4342	TCGArg a1, TCGArg a2, TCGCond cond)
4343	{
4344	/*
4345	* Without AVX512, there are no 64-bit unsigned comparisons.
4346	* We must bias the inputs so that they become signed.
4347	* All other swapping and inversion are handled during code generation.
4348	*/
4349	if (vece == MO_64 && !have_avx512dq && is_unsigned_cond(cond)) {
4350	TCGv_vec v1 = temp_tcgv_vec(arg_temp(*a1));
4351	TCGv_vec v2 = temp_tcgv_vec(arg_temp(*a2));
4352	TCGv_vec t1 = tcg_temp_new_vec(type);
4353	TCGv_vec t2 = tcg_temp_new_vec(type);
4354	TCGv_vec t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1));
4355
4356	tcg_gen_sub_vec(vece, t1, v1, t3);
4357	tcg_gen_sub_vec(vece, t2, v2, t3);
4358	*a1 = tcgv_vec_arg(t1);
4359	*a2 = tcgv_vec_arg(t2);
4360	cond = tcg_signed_cond(cond);
4361	}
4362	return cond;
4363	}
4364
4365	static void expand_vec_cmp(TCGType type, unsigned vece, TCGArg a0,
4366	TCGArg a1, TCGArg a2, TCGCond cond)
4367	{
4368	cond = expand_vec_cond(type, vece, &a1, &a2, cond);
4369	/* Expand directly; do not recurse. */
4370	vec_gen_4(INDEX_op_cmp_vec, type, vece, a0, a1, a2, cond);
4371	}
4372
4373	static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGArg a0,
4374	TCGArg a1, TCGArg a2,
4375	TCGArg a3, TCGArg a4, TCGCond cond)
4376	{
4377	cond = expand_vec_cond(type, vece, &a1, &a2, cond);
4378	/* Expand directly; do not recurse. */
4379	vec_gen_6(INDEX_op_cmpsel_vec, type, vece, a0, a1, a2, a3, a4, cond);
4380	}
4381
4382	void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
4383	TCGArg a0, ...)
4384	{
4385	va_list va;
4386	TCGArg a1, a2, a3, a4, a5;
4387	TCGv_vec v0, v1, v2;
4388
4389	va_start(va, a0);
4390	a1 = va_arg(va, TCGArg);
4391	a2 = va_arg(va, TCGArg);
4392	v0 = temp_tcgv_vec(arg_temp(a0));
4393	v1 = temp_tcgv_vec(arg_temp(a1));
4394
4395	switch (opc) {
4396	case INDEX_op_shli_vec:
4397	expand_vec_shi(type, vece, false, v0, v1, a2);
4398	break;
4399	case INDEX_op_shri_vec:
4400	expand_vec_shi(type, vece, true, v0, v1, a2);
4401	break;
4402	case INDEX_op_sari_vec:
4403	expand_vec_sari(type, vece, v0, v1, a2);
4404	break;
4405
4406	case INDEX_op_rotli_vec:
4407	expand_vec_rotli(type, vece, v0, v1, a2);
4408	break;
4409
4410	case INDEX_op_rotls_vec:
4411	expand_vec_rotls(type, vece, v0, v1, temp_tcgv_i32(arg_temp(a2)));
4412	break;
4413
4414	case INDEX_op_rotlv_vec:
4415	v2 = temp_tcgv_vec(arg_temp(a2));
4416	expand_vec_rotv(type, vece, v0, v1, v2, false);
4417	break;
4418	case INDEX_op_rotrv_vec:
4419	v2 = temp_tcgv_vec(arg_temp(a2));
4420	expand_vec_rotv(type, vece, v0, v1, v2, true);
4421	break;
4422
4423	case INDEX_op_mul_vec:
4424	v2 = temp_tcgv_vec(arg_temp(a2));
4425	expand_vec_mul(type, vece, v0, v1, v2);
4426	break;
4427
4428	case INDEX_op_cmp_vec:
4429	a3 = va_arg(va, TCGArg);
4430	expand_vec_cmp(type, vece, a0, a1, a2, a3);
4431	break;
4432
4433	case INDEX_op_cmpsel_vec:
4434	a3 = va_arg(va, TCGArg);
4435	a4 = va_arg(va, TCGArg);
4436	a5 = va_arg(va, TCGArg);
4437	expand_vec_cmpsel(type, vece, a0, a1, a2, a3, a4, a5);
4438	break;
4439
4440	default:
4441	break;
4442	}
4443
4444	va_end(va);
4445	}
4446
4447	static const int tcg_target_callee_save_regs[] = {
4448	TCG_REG_RBP,
4449	TCG_REG_RBX,
4450	#if defined(_WIN64)
4451	TCG_REG_RDI,
4452	TCG_REG_RSI,
4453	#endif
4454	TCG_REG_R12,
4455	TCG_REG_R13,
4456	TCG_REG_R14, /* Currently used for the global env. */
4457	TCG_REG_R15,
4458	};
4459
4460	/* Compute frame size via macros, to share between tcg_target_qemu_prologue
4461	and tcg_register_jit. */
4462
4463	#define PUSH_SIZE \
4464	((1 + ARRAY_SIZE(tcg_target_callee_save_regs)) * sizeof(tcg_target_long))
4465
4466	#define FRAME_SIZE \
4467	((PUSH_SIZE \
4468	+ TCG_STATIC_CALL_ARGS_SIZE \
4469	+ CPU_TEMP_BUF_NLONGS * sizeof(long) \
4470	+ TCG_TARGET_STACK_ALIGN - 1) \
4471	& ~(TCG_TARGET_STACK_ALIGN - 1))
4472
4473	/* Generate global QEMU prologue and epilogue code */
4474	static void tcg_target_qemu_prologue(TCGContext *s)
4475	{
4476	int i, stack_addend;
4477
4478	/* TB prologue */
4479
4480	/* Reserve some stack space, also for TCG temps. */
4481	stack_addend = FRAME_SIZE - PUSH_SIZE;
4482	tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
4483	CPU_TEMP_BUF_NLONGS * sizeof(long));
4484
4485	/* Save all callee saved registers. */
4486	for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
4487	tcg_out_push(s, tcg_target_callee_save_regs[i]);
4488	}
4489
4490	if (!tcg_use_softmmu && guest_base) {
4491	int seg = setup_guest_base_seg();
4492	if (seg != 0) {
4493	x86_guest_base.seg = seg;
4494	} else if (guest_base == (int32_t)guest_base) {
4495	x86_guest_base.ofs = guest_base;
4496	} else {
4497	/* Choose R12 because, as a base, it requires a SIB byte. */
4498	x86_guest_base.index = TCG_REG_R12;
4499	tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base.index, guest_base);
4500	tcg_regset_set_reg(s->reserved_regs, x86_guest_base.index);
4501	}
4502	}
4503
4504	tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
4505	tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
4506	/* jmp tb. /
4507	tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]);
4508
4509	/*
4510	* Return path for goto_ptr. Set return value to 0, a-la exit_tb,
4511	* and fall through to the rest of the epilogue.
4512	*/
4513	tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
4514	tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0);
4515
4516	/* TB epilogue */
4517	tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
4518
4519	tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend);
4520
4521	if (have_avx2) {
4522	tcg_out_vex_opc(s, OPC_VZEROUPPER, 0, 0, 0, 0);
4523	}
4524	for (i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) {
4525	tcg_out_pop(s, tcg_target_callee_save_regs[i]);
4526	}
4527	tcg_out_opc(s, OPC_RET, 0, 0, 0);
4528	}
4529
4530	static void tcg_out_tb_start(TCGContext *s)
4531	{
4532	/* nothing to do */
4533	}
4534
4535	static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
4536	{
4537	memset(p, 0x90, count);
4538	}
4539
4540	static void tcg_target_init(TCGContext *s)
4541	{
4542	tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS;
4543	tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS;
4544	if (have_avx1) {
4545	tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS;
4546	tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS;
4547	}
4548	if (have_avx2) {
4549	tcg_target_available_regs[TCG_TYPE_V256] = ALL_VECTOR_REGS;
4550	}
4551
4552	tcg_target_call_clobber_regs = ALL_VECTOR_REGS;
4553	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EAX);
4554	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EDX);
4555	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_ECX);
4556	#if !defined(_WIN64)
4557	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RDI);
4558	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RSI);
4559	#endif
4560	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
4561	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
4562	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
4563	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
4564
4565	s->reserved_regs = 0;
4566	tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
4567	tcg_regset_set_reg(s->reserved_regs, TCG_TMP_VEC);
4568	#ifdef _WIN64
4569	/* These are call saved, and we don't save them, so don't use them. */
4570	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM6);
4571	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM7);
4572	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM8);
4573	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM9);
4574	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM10);
4575	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM11);
4576	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM12);
4577	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM13);
4578	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM14);
4579	tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM15);
4580	#endif
4581	}
4582
4583	typedef struct {
4584	DebugFrameHeader h;
4585	uint8_t fde_def_cfa[4];
4586	uint8_t fde_reg_ofs[14];
4587	} DebugFrame;
4588
4589	/* We're expecting a 2 byte uleb128 encoded value. */
4590	QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
4591
4592	#ifdef __ELF__
4593	#define ELF_HOST_MACHINE EM_X86_64
4594
4595	static const DebugFrame debug_frame = {
4596	.h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
4597	.h.cie.id = -1,
4598	.h.cie.version = 1,
4599	.h.cie.code_align = 1,
4600	.h.cie.data_align = 0x78, /* sleb128 -8 */
4601	.h.cie.return_column = 16,
4602
4603	/* Total FDE size does not include the "len" member. */
4604	.h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
4605
4606	.fde_def_cfa = {
4607	12, 7, /* DW_CFA_def_cfa %rsp, ... */
4608	(FRAME_SIZE & 0x7f) \| 0x80, /* ... uleb128 FRAME_SIZE */
4609	(FRAME_SIZE >> 7)
4610	},
4611	.fde_reg_ofs = {
4612	0x90, 1, /* DW_CFA_offset, %rip, -8 */
4613	/* The following ordering must match tcg_target_callee_save_regs. */
4614	0x86, 2, /* DW_CFA_offset, %rbp, -16 */
4615	0x83, 3, /* DW_CFA_offset, %rbx, -24 */
4616	0x8c, 4, /* DW_CFA_offset, %r12, -32 */
4617	0x8d, 5, /* DW_CFA_offset, %r13, -40 */
4618	0x8e, 6, /* DW_CFA_offset, %r14, -48 */
4619	0x8f, 7, /* DW_CFA_offset, %r15, -56 */
4620	}
4621	};
4622
4623	void tcg_register_jit(const void *buf, size_t buf_size)
4624	{
4625	tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
4626	}
4627	#endif