emit.c.inc · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

inc 4,822 lines 150 KB

Raw

1	/*
2	* New-style TCG opcode generator for i386 instructions
3	*
4	* Copyright (c) 2022 Red Hat, Inc.
5	*
6	* Author: Paolo Bonzini <pbonzini@redhat.com>
7	*
8	* This library is free software; you can redistribute it and/or
9	* modify it under the terms of the GNU Lesser General Public
10	* License as published by the Free Software Foundation; either
11	* version 2.1 of the License, or (at your option) any later version.
12	*
13	* This library is distributed in the hope that it will be useful,
14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16	* Lesser General Public License for more details.
17	*
18	* You should have received a copy of the GNU Lesser General Public
19	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
20	*/
21
22	#define MMX_OFFSET(reg) \
23	({ assert((reg) >= 0 && (reg) <= 7); \
24	offsetof(CPUX86State, fpregs[reg].mmx); })
25
26	#define ZMM_OFFSET(reg) \
27	({ assert((reg) >= 0 && (reg) <= 15); \
28	offsetof(CPUX86State, xmm_regs[reg]); })
29
30	typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg);
31	typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg);
32	typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b);
33	typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
34	TCGv_ptr reg_c);
35	typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
36	TCGv_ptr reg_c, TCGv_ptr reg_d);
37	typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
38	TCGv_i32 val);
39	typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
40	TCGv_ptr reg_c, TCGv_i32 val);
41	typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val);
42	typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c,
43	TCGv_i32 val);
44	typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
45	TCGv val);
46	typedef void (*SSEFunc_0_eppptit)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
47	TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale, TCGv amask);
48	typedef void (*SSEFunc_0_eppppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
49	TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 flags);
50	typedef void (*SSEFunc_0_eppppii)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
51	TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 even,
52	TCGv_i32 odd);
53
54	static void gen_JMP_m(DisasContext s, X86DecodedInsn decode);
55	static void gen_JMP(DisasContext s, X86DecodedInsn decode);
56
57	static inline TCGv_i32 tcg_constant8u_i32(uint8_t val)
58	{
59	return tcg_constant_i32(val);
60	}
61
62	static void gen_NM_exception(DisasContext *s)
63	{
64	gen_exception(s, EXCP07_PREX);
65	}
66
67	static void gen_lea_modrm(DisasContext s, X86DecodedInsn decode)
68	{
69	AddressParts *mem = &decode->mem;
70	TCGv ea;
71
72	ea = gen_lea_modrm_1(s, *mem, decode->e.vex_class == 12);
73	if (decode->e.special == X86_SPECIAL_BitTest) {
74	MemOp ot = decode->op[1].ot;
75	int poslen = 8 << ot;
76	int opn = decode->op[2].n;
77	TCGv ofs = tcg_temp_new();
78
79	/* Extract memory displacement from the second operand. */
80	assert(decode->op[2].unit == X86_OP_INT && decode->op[2].ot != MO_8);
81	tcg_gen_sextract_tl(ofs, cpu_regs[opn], 3, poslen - 3);
82	tcg_gen_andi_tl(ofs, ofs, -1 << ot);
83	tcg_gen_add_tl(s->A0, ea, ofs);
84	ea = s->A0;
85	}
86
87	gen_lea_v_seg(s, ea, mem->def_seg, s->override);
88	}
89
90	static inline int mmx_offset(MemOp ot)
91	{
92	switch (ot) {
93	case MO_8:
94	return offsetof(MMXReg, MMX_B(0));
95	case MO_16:
96	return offsetof(MMXReg, MMX_W(0));
97	case MO_32:
98	return offsetof(MMXReg, MMX_L(0));
99	case MO_64:
100	return offsetof(MMXReg, MMX_Q(0));
101	default:
102	g_assert_not_reached();
103	}
104	}
105
106	static inline int xmm_offset(MemOp ot)
107	{
108	switch (ot) {
109	case MO_8:
110	return offsetof(ZMMReg, ZMM_B(0));
111	case MO_16:
112	return offsetof(ZMMReg, ZMM_W(0));
113	case MO_32:
114	return offsetof(ZMMReg, ZMM_L(0));
115	case MO_64:
116	return offsetof(ZMMReg, ZMM_Q(0));
117	case MO_128:
118	return offsetof(ZMMReg, ZMM_X(0));
119	case MO_256:
120	return offsetof(ZMMReg, ZMM_Y(0));
121	default:
122	g_assert_not_reached();
123	}
124	}
125
126	static int vector_reg_offset(X86DecodedOp *op)
127	{
128	assert(op->unit == X86_OP_MMX \|\| op->unit == X86_OP_SSE);
129
130	if (op->unit == X86_OP_MMX) {
131	return op->offset - mmx_offset(op->ot);
132	} else {
133	return op->offset - xmm_offset(op->ot);
134	}
135	}
136
137	static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n)
138	{
139	int base_ofs = vector_reg_offset(op);
140	switch(ot) {
141	case MO_8:
142	if (op->unit == X86_OP_MMX) {
143	return base_ofs + offsetof(MMXReg, MMX_B(n));
144	} else {
145	return base_ofs + offsetof(ZMMReg, ZMM_B(n));
146	}
147	case MO_16:
148	if (op->unit == X86_OP_MMX) {
149	return base_ofs + offsetof(MMXReg, MMX_W(n));
150	} else {
151	return base_ofs + offsetof(ZMMReg, ZMM_W(n));
152	}
153	case MO_32:
154	if (op->unit == X86_OP_MMX) {
155	return base_ofs + offsetof(MMXReg, MMX_L(n));
156	} else {
157	return base_ofs + offsetof(ZMMReg, ZMM_L(n));
158	}
159	case MO_64:
160	if (op->unit == X86_OP_MMX) {
161	return base_ofs;
162	} else {
163	return base_ofs + offsetof(ZMMReg, ZMM_Q(n));
164	}
165	case MO_128:
166	assert(op->unit == X86_OP_SSE);
167	return base_ofs + offsetof(ZMMReg, ZMM_X(n));
168	case MO_256:
169	assert(op->unit == X86_OP_SSE);
170	return base_ofs + offsetof(ZMMReg, ZMM_Y(n));
171	default:
172	g_assert_not_reached();
173	}
174	}
175
176	static void compute_mmx_offset(X86DecodedOp *op)
177	{
178	if (!op->has_ea) {
179	op->offset = MMX_OFFSET(op->n) + mmx_offset(op->ot);
180	} else {
181	op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot);
182	}
183	}
184
185	static void compute_xmm_offset(X86DecodedOp *op)
186	{
187	if (!op->has_ea) {
188	op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot);
189	} else {
190	op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot);
191	}
192	}
193
194	static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned)
195	{
196	switch(ot) {
197	case MO_8:
198	gen_op_ld_v(s, MO_8, temp, s->A0);
199	tcg_gen_st8_tl(temp, tcg_env, dest_ofs);
200	break;
201	case MO_16:
202	gen_op_ld_v(s, MO_16, temp, s->A0);
203	tcg_gen_st16_tl(temp, tcg_env, dest_ofs);
204	break;
205	case MO_32:
206	gen_op_ld_v(s, MO_32, temp, s->A0);
207	tcg_gen_st32_tl(temp, tcg_env, dest_ofs);
208	break;
209	case MO_64:
210	gen_ldq_env_A0(s, dest_ofs);
211	break;
212	case MO_128:
213	gen_ldo_env_A0(s, dest_ofs, aligned);
214	break;
215	case MO_256:
216	gen_ldy_env_A0(s, dest_ofs, aligned);
217	break;
218	default:
219	g_assert_not_reached();
220	}
221	}
222
223	static bool sse_needs_alignment(DisasContext s, X86DecodedInsn decode, MemOp ot)
224	{
225	switch (decode->e.vex_class) {
226	case 2:
227	case 4:
228	if ((s->prefix & PREFIX_VEX) \|\|
229	decode->e.vex_special == X86_VEX_SSEUnaligned) {
230	/* MOST legacy SSE instructions require aligned memory operands, but not all. */
231	return false;
232	}
233	/* fall through */
234	case 1:
235	return ot >= MO_128;
236
237	default:
238	return false;
239	}
240	}
241
242	static void gen_load(DisasContext s, X86DecodedInsn decode, int opn, TCGv v)
243	{
244	X86DecodedOp *op = &decode->op[opn];
245
246	switch (op->unit) {
247	case X86_OP_SKIP:
248	return;
249	case X86_OP_SEG:
250	tcg_gen_ld32u_tl(v, tcg_env,
251	offsetof(CPUX86State,segs[op->n].selector));
252	break;
253	#ifndef CONFIG_USER_ONLY
254	case X86_OP_CR:
255	if (op->n == 8) {
256	translator_io_start(&s->base);
257	gen_helper_read_cr8(v, tcg_env);
258	} else {
259	tcg_gen_ld_tl(v, tcg_env, offsetof(CPUX86State, cr[op->n]));
260	}
261	break;
262	case X86_OP_DR:
263	/* CR4.DE tested in the helper. */
264	gen_helper_get_dr(v, tcg_env, tcg_constant_i32(op->n));
265	break;
266	#endif
267	case X86_OP_INT:
268	if (op->has_ea) {
269	if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) {
270	gen_op_ld_v(s, op->ot \| MO_SIGN, v, s->A0);
271	} else {
272	gen_op_ld_v(s, op->ot, v, s->A0);
273	}
274
275	} else if (op->ot < MO_TL && v == s->T0 &&
276	(decode->e.special == X86_SPECIAL_SExtT0 \|\|
277	decode->e.special == X86_SPECIAL_ZExtT0)) {
278	if (op->ot == MO_8 && byte_reg_is_xH(s, op->n)) {
279	if (decode->e.special == X86_SPECIAL_SExtT0) {
280	tcg_gen_sextract_tl(v, cpu_regs[op->n - 4], 8, 8);
281	} else {
282	tcg_gen_extract_tl(v, cpu_regs[op->n - 4], 8, 8);
283	}
284	} else {
285	if (decode->e.special == X86_SPECIAL_SExtT0) {
286	tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot \| MO_SIGN);
287	} else {
288	tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot);
289	}
290	}
291
292	} else {
293	gen_op_mov_v_reg(s, op->ot, v, op->n);
294	}
295	break;
296	case X86_OP_IMM:
297	tcg_gen_movi_tl(v, op->imm);
298	break;
299
300	case X86_OP_MMX:
301	compute_mmx_offset(op);
302	goto load_vector;
303
304	case X86_OP_SSE:
305	compute_xmm_offset(op);
306	load_vector:
307	if (op->has_ea) {
308	bool aligned = sse_needs_alignment(s, decode, op->ot);
309	gen_load_sse(s, v, op->ot, op->offset, aligned);
310	}
311	break;
312
313	default:
314	g_assert_not_reached();
315	}
316	}
317
318	static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn)
319	{
320	X86DecodedOp *op = &decode->op[opn];
321
322	assert(op->unit == X86_OP_MMX \|\| op->unit == X86_OP_SSE);
323	if (op->v_ptr) {
324	return op->v_ptr;
325	}
326	op->v_ptr = tcg_temp_new_ptr();
327
328	/* The temporary points to the MMXReg or ZMMReg. */
329	tcg_gen_addi_ptr(op->v_ptr, tcg_env, vector_reg_offset(op));
330	return op->v_ptr;
331	}
332
333	#define OP_PTR0 op_ptr(decode, 0)
334	#define OP_PTR1 op_ptr(decode, 1)
335	#define OP_PTR2 op_ptr(decode, 2)
336
337	static void gen_writeback(DisasContext s, X86DecodedInsn decode, int opn, TCGv v)
338	{
339	X86DecodedOp *op = &decode->op[opn];
340	switch (op->unit) {
341	case X86_OP_SKIP:
342	break;
343	case X86_OP_SEG:
344	/* Note that gen_movl_seg takes care of interrupt shadow and TF. */
345	gen_movl_seg(s, op->n, v, op->n == R_SS);
346	break;
347	case X86_OP_INT:
348	if (op->has_ea) {
349	gen_op_st_v(s, op->ot, v, s->A0);
350	} else {
351	gen_op_mov_reg_v(s, op->ot, op->n, v);
352	}
353	break;
354	case X86_OP_MMX:
355	break;
356	case X86_OP_SSE:
357	if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) {
358	tcg_gen_gvec_dup_imm(MO_64,
359	offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)),
360	16, 16, 0);
361	}
362	break;
363	#ifndef CONFIG_USER_ONLY
364	case X86_OP_CR:
365	if (op->n == 8) {
366	translator_io_start(&s->base);
367	}
368	gen_helper_write_crN(tcg_env, tcg_constant_i32(op->n), v);
369	s->base.is_jmp = DISAS_EOB_NEXT;
370	break;
371	case X86_OP_DR:
372	/* CR4.DE tested in the helper. */
373	gen_helper_set_dr(tcg_env, tcg_constant_i32(op->n), v);
374	s->base.is_jmp = DISAS_EOB_NEXT;
375	break;
376	#endif
377	default:
378	g_assert_not_reached();
379	}
380	op->unit = X86_OP_SKIP;
381	}
382
383	static inline int vector_len(DisasContext s, X86DecodedInsn decode)
384	{
385	if (decode->e.special == X86_SPECIAL_MMX &&
386	!(s->prefix & (PREFIX_DATA \| PREFIX_REPZ \| PREFIX_REPNZ))) {
387	return 8;
388	}
389	return s->vex_l ? 32 : 16;
390	}
391
392	static void prepare_update1_cc(X86DecodedInsn decode, DisasContext s, CCOp op)
393	{
394	decode->cc_dst = s->T0;
395	decode->cc_op = op;
396	}
397
398	static void prepare_update2_cc(X86DecodedInsn decode, DisasContext s, CCOp op)
399	{
400	decode->cc_src = s->T1;
401	decode->cc_dst = s->T0;
402	decode->cc_op = op;
403	}
404
405	static void prepare_update_cc_incdec(X86DecodedInsn decode, DisasContext s, CCOp op)
406	{
407	gen_compute_eflags_c(s, s->T1);
408	prepare_update2_cc(decode, s, op);
409	}
410
411	static void prepare_update3_cc(X86DecodedInsn decode, DisasContext s, CCOp op, TCGv reg)
412	{
413	decode->cc_src2 = reg;
414	decode->cc_src = s->T1;
415	decode->cc_dst = s->T0;
416	decode->cc_op = op;
417	}
418
419	/* Set up decode->cc_* to modify CF while keeping other flags unchanged. */
420	static void prepare_update_cf(X86DecodedInsn decode, DisasContext s, TCGv cf)
421	{
422	switch (s->cc_op) {
423	case CC_OP_ADOX:
424	case CC_OP_ADCOX:
425	decode->cc_src2 = cpu_cc_src2;
426	decode->cc_src = cpu_cc_src;
427	decode->cc_op = CC_OP_ADCOX;
428	break;
429
430	case CC_OP_EFLAGS:
431	case CC_OP_ADCX:
432	decode->cc_src = cpu_cc_src;
433	decode->cc_op = CC_OP_ADCX;
434	break;
435
436	default:
437	decode->cc_src = tcg_temp_new();
438	gen_mov_eflags(s, decode->cc_src);
439	decode->cc_op = CC_OP_ADCX;
440	break;
441	}
442	decode->cc_dst = cf;
443	}
444
445	static void gen_store_sse(DisasContext s, X86DecodedInsn decode, int src_ofs)
446	{
447	MemOp ot = decode->op[0].ot;
448	int vec_len = vector_len(s, decode);
449	bool aligned = sse_needs_alignment(s, decode, ot);
450
451	if (!decode->op[0].has_ea) {
452	tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len);
453	return;
454	}
455
456	switch (ot) {
457	case MO_64:
458	gen_stq_env_A0(s, src_ofs);
459	break;
460	case MO_128:
461	gen_sto_env_A0(s, src_ofs, aligned);
462	break;
463	case MO_256:
464	gen_sty_env_A0(s, src_ofs, aligned);
465	break;
466	default:
467	g_assert_not_reached();
468	}
469	}
470
471	static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b)
472	{
473	gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b);
474	}
475
476	#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1)
477	static const SSEFunc_0_epp fns_3dnow[] = {
478	[0x0c] = gen_helper_pi2fw,
479	[0x0d] = gen_helper_pi2fd,
480	[0x1c] = gen_helper_pf2iw,
481	[0x1d] = gen_helper_pf2id,
482	[0x8a] = gen_helper_pfnacc,
483	[0x8e] = gen_helper_pfpnacc,
484	[0x90] = gen_helper_pfcmpge,
485	[0x94] = gen_helper_pfmin,
486	[0x96] = gen_helper_pfrcp,
487	[0x97] = gen_helper_pfrsqrt,
488	[0x9a] = gen_helper_pfsub,
489	[0x9e] = gen_helper_pfadd,
490	[0xa0] = gen_helper_pfcmpgt,
491	[0xa4] = gen_helper_pfmax,
492	[0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */
493	[0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */
494	[0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */
495	[0xaa] = gen_helper_pfsubr,
496	[0xae] = gen_helper_pfacc,
497	[0xb0] = gen_helper_pfcmpeq,
498	[0xb4] = gen_helper_pfmul,
499	[0xb7] = gen_helper_pmulhrw_mmx,
500	[0xbb] = gen_helper_pswapd,
501	[0xbf] = gen_helper_pavgusb,
502	};
503
504	static void gen_3dnow(DisasContext s, X86DecodedInsn decode)
505	{
506	uint8_t b = decode->immediate;
507	SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL;
508
509	if (!fn) {
510	gen_illegal_opcode(s);
511	return;
512	}
513	if (s->flags & HF_TS_MASK) {
514	gen_NM_exception(s);
515	return;
516	}
517	if (s->flags & HF_EM_MASK) {
518	gen_illegal_opcode(s);
519	return;
520	}
521
522	gen_helper_enter_mmx(tcg_env);
523	if (fn == FN_3DNOW_MOVE) {
524	TCGv_i64 t = tcg_temp_new_i64();
525
526	tcg_gen_ld_i64(t, tcg_env, decode->op[1].offset);
527	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset);
528	} else {
529	fn(tcg_env, OP_PTR0, OP_PTR1);
530	}
531	}
532
533	/*
534	* 00 = v*ps Vps, Hps, Wpd
535	* 66 = v*pd Vpd, Hpd, Wps
536	* f3 = v*ss Vss, Hss, Wps
537	* f2 = v*sd Vsd, Hsd, Wps
538	*/
539	static inline void gen_unary_fp_sse(DisasContext s, X86DecodedInsn decode,
540	SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm,
541	SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm,
542	SSEFunc_0_eppp sd, SSEFunc_0_eppp ss)
543	{
544	if ((s->prefix & (PREFIX_REPZ \| PREFIX_REPNZ)) != 0) {
545	SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd;
546	if (!fn) {
547	gen_illegal_opcode(s);
548	return;
549	}
550	fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
551	} else {
552	SSEFunc_0_epp ps, pd, fn;
553	ps = s->vex_l ? ps_ymm : ps_xmm;
554	pd = s->vex_l ? pd_ymm : pd_xmm;
555	fn = s->prefix & PREFIX_DATA ? pd : ps;
556	if (!fn) {
557	gen_illegal_opcode(s);
558	return;
559	}
560	fn(tcg_env, OP_PTR0, OP_PTR2);
561	}
562	}
563	#define UNARY_FP_SSE(uname, lname) \
564	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
565	{ \
566	gen_unary_fp_sse(s, decode, \
567	gen_helper_##lname##pd_xmm, \
568	gen_helper_##lname##ps_xmm, \
569	gen_helper_##lname##pd_ymm, \
570	gen_helper_##lname##ps_ymm, \
571	gen_helper_##lname##sd, \
572	gen_helper_##lname##ss); \
573	}
574	UNARY_FP_SSE(VSQRT, sqrt)
575
576	/*
577	* 00 = v*ps Vps, Hps, Wpd
578	* 66 = v*pd Vpd, Hpd, Wps
579	* f3 = v*ss Vss, Hss, Wps
580	* f2 = v*sd Vsd, Hsd, Wps
581	*/
582	static inline void gen_fp_sse(DisasContext s, X86DecodedInsn decode,
583	SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm,
584	SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm,
585	SSEFunc_0_eppp sd, SSEFunc_0_eppp ss)
586	{
587	SSEFunc_0_eppp ps, pd, fn;
588	if ((s->prefix & (PREFIX_REPZ \| PREFIX_REPNZ)) != 0) {
589	fn = s->prefix & PREFIX_REPZ ? ss : sd;
590	} else {
591	ps = s->vex_l ? ps_ymm : ps_xmm;
592	pd = s->vex_l ? pd_ymm : pd_xmm;
593	fn = s->prefix & PREFIX_DATA ? pd : ps;
594	}
595	if (fn) {
596	fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
597	} else {
598	gen_illegal_opcode(s);
599	}
600	}
601
602	#define FP_SSE(uname, lname) \
603	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
604	{ \
605	gen_fp_sse(s, decode, \
606	gen_helper_##lname##pd_xmm, \
607	gen_helper_##lname##ps_xmm, \
608	gen_helper_##lname##pd_ymm, \
609	gen_helper_##lname##ps_ymm, \
610	gen_helper_##lname##sd, \
611	gen_helper_##lname##ss); \
612	}
613	FP_SSE(VADD, add)
614	FP_SSE(VMUL, mul)
615	FP_SSE(VSUB, sub)
616	FP_SSE(VMIN, min)
617	FP_SSE(VDIV, div)
618	FP_SSE(VMAX, max)
619
620	#define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd) \
621	static void gen_##uname##Px(DisasContext s, X86DecodedInsn decode) \
622	{ \
623	SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \
624	SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \
625	SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm; \
626	\
627	fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \
628	tcg_constant_i32(even), \
629	tcg_constant_i32((even) ^ (odd))); \
630	}
631
632	#define FMA_SSE(uname, ptr0, ptr1, ptr2, flags) \
633	FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags) \
634	static void gen_##uname##Sx(DisasContext s, X86DecodedInsn decode) \
635	{ \
636	SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss; \
637	\
638	fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \
639	tcg_constant_i32(flags)); \
640	} \
641
642	FMA_SSE(VFMADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0)
643	FMA_SSE(VFMADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0)
644	FMA_SSE(VFMADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0)
645
646	FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product)
647	FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product)
648	FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product)
649
650	FMA_SSE(VFMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c)
651	FMA_SSE(VFMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c)
652	FMA_SSE(VFMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c)
653
654	FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c\|float_muladd_negate_product)
655	FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c\|float_muladd_negate_product)
656	FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c\|float_muladd_negate_product)
657
658	FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0)
659	FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0)
660	FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0)
661
662	FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c)
663	FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c)
664	FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c)
665
666	#define FP_UNPACK_SSE(uname, lname) \
667	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
668	{ \
669	/* PS maps to the DQ integer instruction, PD maps to QDQ. */ \
670	gen_fp_sse(s, decode, \
671	gen_helper_##lname##qdq_xmm, \
672	gen_helper_##lname##dq_xmm, \
673	gen_helper_##lname##qdq_ymm, \
674	gen_helper_##lname##dq_ymm, \
675	NULL, NULL); \
676	}
677	FP_UNPACK_SSE(VUNPCKLPx, punpckl)
678	FP_UNPACK_SSE(VUNPCKHPx, punpckh)
679
680	/*
681	* 00 = v*ps Vps, Wpd
682	* f3 = v*ss Vss, Wps
683	*/
684	static inline void gen_unary_fp32_sse(DisasContext s, X86DecodedInsn decode,
685	SSEFunc_0_epp ps_xmm,
686	SSEFunc_0_epp ps_ymm,
687	SSEFunc_0_eppp ss)
688	{
689	if ((s->prefix & (PREFIX_DATA \| PREFIX_REPNZ)) != 0) {
690	goto illegal_op;
691	} else if (s->prefix & PREFIX_REPZ) {
692	if (!ss) {
693	goto illegal_op;
694	}
695	ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
696	} else {
697	SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm;
698	if (!fn) {
699	goto illegal_op;
700	}
701	fn(tcg_env, OP_PTR0, OP_PTR2);
702	}
703	return;
704
705	illegal_op:
706	gen_illegal_opcode(s);
707	}
708	#define UNARY_FP32_SSE(uname, lname) \
709	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
710	{ \
711	gen_unary_fp32_sse(s, decode, \
712	gen_helper_##lname##ps_xmm, \
713	gen_helper_##lname##ps_ymm, \
714	gen_helper_##lname##ss); \
715	}
716	UNARY_FP32_SSE(VRSQRT, rsqrt)
717	UNARY_FP32_SSE(VRCP, rcp)
718
719	/*
720	* 66 = v*pd Vpd, Hpd, Wpd
721	* f2 = v*ps Vps, Hps, Wps
722	*/
723	static inline void gen_horizontal_fp_sse(DisasContext s, X86DecodedInsn decode,
724	SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm,
725	SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm)
726	{
727	SSEFunc_0_eppp ps, pd, fn;
728	ps = s->vex_l ? ps_ymm : ps_xmm;
729	pd = s->vex_l ? pd_ymm : pd_xmm;
730	fn = s->prefix & PREFIX_DATA ? pd : ps;
731	fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
732	}
733	#define HORIZONTAL_FP_SSE(uname, lname) \
734	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
735	{ \
736	gen_horizontal_fp_sse(s, decode, \
737	gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \
738	gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \
739	}
740	HORIZONTAL_FP_SSE(VHADD, hadd)
741	HORIZONTAL_FP_SSE(VHSUB, hsub)
742	HORIZONTAL_FP_SSE(VADDSUB, addsub)
743
744	static inline void gen_ternary_sse(DisasContext s, X86DecodedInsn decode,
745	int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm)
746	{
747	SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm;
748	TCGv_ptr ptr3 = tcg_temp_new_ptr();
749
750	/* The format of the fourth input is Lx */
751	tcg_gen_addi_ptr(ptr3, tcg_env, ZMM_OFFSET(op3));
752	fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3);
753	}
754	#define TERNARY_SSE(uname, uvname, lname) \
755	static void gen_##uvname(DisasContext s, X86DecodedInsn decode) \
756	{ \
757	gen_ternary_sse(s, decode, (uint8_t)decode->immediate >> 4, \
758	gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \
759	} \
760	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
761	{ \
762	gen_ternary_sse(s, decode, 0, \
763	gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \
764	}
765	TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps)
766	TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd)
767	TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb)
768
769	static inline void gen_binary_imm_sse(DisasContext s, X86DecodedInsn decode,
770	SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm)
771	{
772	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
773	if (!s->vex_l) {
774	xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
775	} else {
776	ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
777	}
778	}
779
780	#define BINARY_IMM_SSE(uname, lname) \
781	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
782	{ \
783	gen_binary_imm_sse(s, decode, \
784	gen_helper_##lname##_xmm, \
785	gen_helper_##lname##_ymm); \
786	}
787
788	BINARY_IMM_SSE(VBLENDPD, blendpd)
789	BINARY_IMM_SSE(VBLENDPS, blendps)
790	BINARY_IMM_SSE(VPBLENDW, pblendw)
791	BINARY_IMM_SSE(VDPPS, dpps)
792	#define gen_helper_dppd_ymm NULL
793	BINARY_IMM_SSE(VDPPD, dppd)
794	BINARY_IMM_SSE(VMPSADBW, mpsadbw)
795	BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq)
796
797
798	#define UNARY_INT_GVEC(uname, func, ...) \
799	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
800	{ \
801	int vec_len = vector_len(s, decode); \
802	\
803	func(__VA_ARGS__, decode->op[0].offset, \
804	decode->op[2].offset, vec_len, vec_len); \
805	}
806	UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8)
807	UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16)
808	UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32)
809	UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128)
810	UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8)
811	UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16)
812	UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32)
813	UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64)
814
815
816	#define BINARY_INT_GVEC(uname, func, ...) \
817	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
818	{ \
819	int vec_len = vector_len(s, decode); \
820	\
821	func(__VA_ARGS__, \
822	decode->op[0].offset, decode->op[1].offset, \
823	decode->op[2].offset, vec_len, vec_len); \
824	}
825
826	BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8)
827	BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16)
828	BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32)
829	BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64)
830	BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8)
831	BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16)
832	BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8)
833	BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16)
834	BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64)
835	BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8)
836	BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32)
837	BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16)
838	BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64)
839	BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8)
840	BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16)
841	BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32)
842	BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64)
843	BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8)
844	BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16)
845	BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32)
846	BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8)
847	BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16)
848	BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32)
849	BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8)
850	BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16)
851	BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32)
852	BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8)
853	BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16)
854	BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32)
855	BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16)
856	BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32)
857	BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64)
858	BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8)
859	BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16)
860	BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32)
861	BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64)
862	BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8)
863	BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16)
864	BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8)
865	BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16)
866	BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64)
867
868
869	/*
870	* 00 = p* Pq, Qq (if mmx not NULL; no VEX)
871	* 66 = vp* Vx, Hx, Wx
872	*
873	* These are really the same encoding, because 1) V is the same as P when VEX.V
874	* is not present 2) P and Q are the same as H and W apart from MM/XMM
875	*/
876	static inline void gen_binary_int_sse(DisasContext s, X86DecodedInsn decode,
877	SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm)
878	{
879	assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX));
880
881	if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) {
882	/* VEX encoding is not applicable to MMX instructions. */
883	gen_illegal_opcode(s);
884	return;
885	}
886	if (!(s->prefix & PREFIX_DATA)) {
887	mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
888	} else if (!s->vex_l) {
889	xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
890	} else {
891	ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
892	}
893	}
894
895
896	#define BINARY_INT_MMX(uname, lname) \
897	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
898	{ \
899	gen_binary_int_sse(s, decode, \
900	gen_helper_##lname##_mmx, \
901	gen_helper_##lname##_xmm, \
902	gen_helper_##lname##_ymm); \
903	}
904	BINARY_INT_MMX(PUNPCKLBW, punpcklbw)
905	BINARY_INT_MMX(PUNPCKLWD, punpcklwd)
906	BINARY_INT_MMX(PUNPCKLDQ, punpckldq)
907	BINARY_INT_MMX(PACKSSWB, packsswb)
908	BINARY_INT_MMX(PACKUSWB, packuswb)
909	BINARY_INT_MMX(PUNPCKHBW, punpckhbw)
910	BINARY_INT_MMX(PUNPCKHWD, punpckhwd)
911	BINARY_INT_MMX(PUNPCKHDQ, punpckhdq)
912	BINARY_INT_MMX(PACKSSDW, packssdw)
913
914	BINARY_INT_MMX(PAVGB, pavgb)
915	BINARY_INT_MMX(PAVGW, pavgw)
916	BINARY_INT_MMX(PMADDWD, pmaddwd)
917	BINARY_INT_MMX(PMULHUW, pmulhuw)
918	BINARY_INT_MMX(PMULHW, pmulhw)
919	BINARY_INT_MMX(PMULUDQ, pmuludq)
920	BINARY_INT_MMX(PSADBW, psadbw)
921
922	BINARY_INT_MMX(PSLLW_r, psllw)
923	BINARY_INT_MMX(PSLLD_r, pslld)
924	BINARY_INT_MMX(PSLLQ_r, psllq)
925	BINARY_INT_MMX(PSRLW_r, psrlw)
926	BINARY_INT_MMX(PSRLD_r, psrld)
927	BINARY_INT_MMX(PSRLQ_r, psrlq)
928	BINARY_INT_MMX(PSRAW_r, psraw)
929	BINARY_INT_MMX(PSRAD_r, psrad)
930
931	BINARY_INT_MMX(PHADDW, phaddw)
932	BINARY_INT_MMX(PHADDSW, phaddsw)
933	BINARY_INT_MMX(PHADDD, phaddd)
934	BINARY_INT_MMX(PHSUBW, phsubw)
935	BINARY_INT_MMX(PHSUBSW, phsubsw)
936	BINARY_INT_MMX(PHSUBD, phsubd)
937	BINARY_INT_MMX(PMADDUBSW, pmaddubsw)
938	BINARY_INT_MMX(PSHUFB, pshufb)
939	BINARY_INT_MMX(PSIGNB, psignb)
940	BINARY_INT_MMX(PSIGNW, psignw)
941	BINARY_INT_MMX(PSIGND, psignd)
942	BINARY_INT_MMX(PMULHRSW, pmulhrsw)
943
944	/* Instructions with no MMX equivalent. */
945	#define BINARY_INT_SSE(uname, lname) \
946	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
947	{ \
948	gen_binary_int_sse(s, decode, \
949	NULL, \
950	gen_helper_##lname##_xmm, \
951	gen_helper_##lname##_ymm); \
952	}
953
954	/* Instructions with no MMX equivalent. */
955	BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq)
956	BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq)
957	BINARY_INT_SSE(VPACKUSDW, packusdw)
958	BINARY_INT_SSE(VPERMILPS, vpermilps)
959	BINARY_INT_SSE(VPERMILPD, vpermilpd)
960	BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd)
961	BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq)
962
963	BINARY_INT_SSE(PMULDQ, pmuldq)
964
965	BINARY_INT_SSE(VAESDEC, aesdec)
966	BINARY_INT_SSE(VAESDECLAST, aesdeclast)
967	BINARY_INT_SSE(VAESENC, aesenc)
968	BINARY_INT_SSE(VAESENCLAST, aesenclast)
969
970	#define UNARY_CMP_SSE(uname, lname) \
971	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
972	{ \
973	if (!s->vex_l) { \
974	gen_helper_##lname##_xmm(tcg_env, OP_PTR1, OP_PTR2); \
975	} else { \
976	gen_helper_##lname##_ymm(tcg_env, OP_PTR1, OP_PTR2); \
977	} \
978	assume_cc_op(s, CC_OP_EFLAGS); \
979	}
980	UNARY_CMP_SSE(VPTEST, ptest)
981	UNARY_CMP_SSE(VTESTPS, vtestps)
982	UNARY_CMP_SSE(VTESTPD, vtestpd)
983
984	static inline void gen_unary_int_sse(DisasContext s, X86DecodedInsn decode,
985	SSEFunc_0_epp xmm, SSEFunc_0_epp ymm)
986	{
987	if (!s->vex_l) {
988	xmm(tcg_env, OP_PTR0, OP_PTR2);
989	} else {
990	ymm(tcg_env, OP_PTR0, OP_PTR2);
991	}
992	}
993
994	#define UNARY_INT_SSE(uname, lname) \
995	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
996	{ \
997	gen_unary_int_sse(s, decode, \
998	gen_helper_##lname##_xmm, \
999	gen_helper_##lname##_ymm); \
1000	}
1001
1002	UNARY_INT_SSE(VPMOVSXBW, pmovsxbw)
1003	UNARY_INT_SSE(VPMOVSXBD, pmovsxbd)
1004	UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq)
1005	UNARY_INT_SSE(VPMOVSXWD, pmovsxwd)
1006	UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq)
1007	UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq)
1008
1009	UNARY_INT_SSE(VPMOVZXBW, pmovzxbw)
1010	UNARY_INT_SSE(VPMOVZXBD, pmovzxbd)
1011	UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq)
1012	UNARY_INT_SSE(VPMOVZXWD, pmovzxwd)
1013	UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq)
1014	UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq)
1015
1016	UNARY_INT_SSE(VMOVSLDUP, pmovsldup)
1017	UNARY_INT_SSE(VMOVSHDUP, pmovshdup)
1018	UNARY_INT_SSE(VMOVDDUP, pmovdldup)
1019
1020	UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd)
1021	UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq)
1022	UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq)
1023	UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps)
1024	UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq)
1025	UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq)
1026	UNARY_INT_SSE(VCVTPH2PS, cvtph2ps)
1027
1028
1029	static inline void gen_unary_imm_sse(DisasContext s, X86DecodedInsn decode,
1030	SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm)
1031	{
1032	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1033	if (!s->vex_l) {
1034	xmm(OP_PTR0, OP_PTR1, imm);
1035	} else {
1036	ymm(OP_PTR0, OP_PTR1, imm);
1037	}
1038	}
1039
1040	#define UNARY_IMM_SSE(uname, lname) \
1041	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
1042	{ \
1043	gen_unary_imm_sse(s, decode, \
1044	gen_helper_##lname##_xmm, \
1045	gen_helper_##lname##_ymm); \
1046	}
1047
1048	UNARY_IMM_SSE(PSHUFD, pshufd)
1049	UNARY_IMM_SSE(PSHUFHW, pshufhw)
1050	UNARY_IMM_SSE(PSHUFLW, pshuflw)
1051	#define gen_helper_vpermq_xmm NULL
1052	UNARY_IMM_SSE(VPERMQ, vpermq)
1053	UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm)
1054	UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm)
1055
1056	static inline void gen_unary_imm_fp_sse(DisasContext s, X86DecodedInsn decode,
1057	SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm)
1058	{
1059	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
1060	if (!s->vex_l) {
1061	xmm(tcg_env, OP_PTR0, OP_PTR1, imm);
1062	} else {
1063	ymm(tcg_env, OP_PTR0, OP_PTR1, imm);
1064	}
1065	}
1066
1067	#define UNARY_IMM_FP_SSE(uname, lname) \
1068	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
1069	{ \
1070	gen_unary_imm_fp_sse(s, decode, \
1071	gen_helper_##lname##_xmm, \
1072	gen_helper_##lname##_ymm); \
1073	}
1074
1075	UNARY_IMM_FP_SSE(VROUNDPS, roundps)
1076	UNARY_IMM_FP_SSE(VROUNDPD, roundpd)
1077
1078	static inline void gen_vexw_avx(DisasContext s, X86DecodedInsn decode,
1079	SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm,
1080	SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm)
1081	{
1082	SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm;
1083	SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm;
1084	SSEFunc_0_eppp fn = s->vex_w ? q : d;
1085	fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
1086	}
1087
1088	/* VEX.W affects whether to operate on 32- or 64-bit elements. */
1089	#define VEXW_AVX(uname, lname) \
1090	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
1091	{ \
1092	gen_vexw_avx(s, decode, \
1093	gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \
1094	gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \
1095	}
1096	VEXW_AVX(VPSLLV, vpsllv)
1097	VEXW_AVX(VPSRLV, vpsrlv)
1098	VEXW_AVX(VPSRAV, vpsrav)
1099	VEXW_AVX(VPMASKMOV, vpmaskmov)
1100
1101	/* Same as above, but with extra arguments to the helper. */
1102	static inline void gen_vsib_avx(DisasContext s, X86DecodedInsn decode,
1103	SSEFunc_0_eppptit d_xmm, SSEFunc_0_eppptit q_xmm,
1104	SSEFunc_0_eppptit d_ymm, SSEFunc_0_eppptit q_ymm)
1105	{
1106	SSEFunc_0_eppptit d = s->vex_l ? d_ymm : d_xmm;
1107	SSEFunc_0_eppptit q = s->vex_l ? q_ymm : q_xmm;
1108	SSEFunc_0_eppptit fn = s->vex_w ? q : d;
1109	TCGv_i32 scale = tcg_constant_i32(decode->mem.scale);
1110	TCGv_ptr index = tcg_temp_new_ptr();
1111	TCGv mask = tcg_constant_tl(MAKE_64BIT_MASK(0, 8 << s->aflag));
1112
1113	/* Pass third input as (index, base, scale) */
1114	tcg_gen_addi_ptr(index, tcg_env, ZMM_OFFSET(decode->mem.index));
1115	fn(tcg_env, OP_PTR0, OP_PTR1, index, s->A0, scale, mask);
1116
1117	/*
1118	* There are two output operands, so zero OP1's high 128 bits
1119	* in the VEX.128 case.
1120	*/
1121	if (!s->vex_l) {
1122	int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1);
1123	tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0);
1124	}
1125	}
1126	#define VSIB_AVX(uname, lname) \
1127	static void gen_##uname(DisasContext s, X86DecodedInsn decode) \
1128	{ \
1129	gen_vsib_avx(s, decode, \
1130	gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \
1131	gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \
1132	}
1133	VSIB_AVX(VPGATHERD, vpgatherd)
1134	VSIB_AVX(VPGATHERQ, vpgatherq)
1135
1136	static void gen_AAA(DisasContext s, X86DecodedInsn decode)
1137	{
1138	gen_update_cc_op(s);
1139	gen_helper_aaa(tcg_env);
1140	assume_cc_op(s, CC_OP_EFLAGS);
1141	}
1142
1143	static void gen_AAD(DisasContext s, X86DecodedInsn decode)
1144	{
1145	gen_helper_aad(s->T0, s->T0, s->T1);
1146	prepare_update1_cc(decode, s, CC_OP_LOGICB);
1147	}
1148
1149	static void gen_AAM(DisasContext s, X86DecodedInsn decode)
1150	{
1151	if (decode->immediate == 0) {
1152	gen_exception(s, EXCP00_DIVZ);
1153	} else {
1154	gen_helper_aam(s->T0, s->T0, s->T1);
1155	prepare_update1_cc(decode, s, CC_OP_LOGICB);
1156	}
1157	}
1158
1159	static void gen_AAS(DisasContext s, X86DecodedInsn decode)
1160	{
1161	gen_update_cc_op(s);
1162	gen_helper_aas(tcg_env);
1163	assume_cc_op(s, CC_OP_EFLAGS);
1164	}
1165
1166	static void gen_ADD(DisasContext s, X86DecodedInsn decode);
1167	static void gen_ADC(DisasContext s, X86DecodedInsn decode)
1168	{
1169	MemOp ot = decode->op[1].ot;
1170	TCGv c_in;
1171
1172	/*
1173	* Try to avoid CC_OP_ADC by transforming as follows:
1174	* CC_ADC: src1 = dst + c_in, src2 = 0, src3 = c_in
1175	* CC_ADD: src1 = dst + c_in, src2 = c_in (no src3)
1176	*
1177	* In general src2 vs. src3 matters when computing AF and OF, but not here:
1178	* - AF is bit 4 of dst^src1^src2, which is bit 4 of dst^src1 in both cases
1179	* - OF is a function of the two MSBs, and in both cases they are zero for src2
1180	*/
1181	if (decode->e.op2 == X86_TYPE_I && decode->immediate == 0) {
1182	gen_compute_eflags_c(s, s->T1);
1183	gen_ADD(s, decode);
1184	return;
1185	}
1186
1187	c_in = tcg_temp_new();
1188	gen_compute_eflags_c(s, c_in);
1189	if (s->prefix & PREFIX_LOCK) {
1190	tcg_gen_add_tl(s->T0, c_in, s->T1);
1191	tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0,
1192	s->mem_index, ot \| MO_LE);
1193	} else {
1194	tcg_gen_add_tl(s->T0, s->T0, s->T1);
1195	tcg_gen_add_tl(s->T0, s->T0, c_in);
1196	}
1197	prepare_update3_cc(decode, s, CC_OP_ADCB + ot, c_in);
1198	}
1199
1200	static void gen_ADCOX(DisasContext s, X86DecodedInsn decode, int cc_op)
1201	{
1202	MemOp ot = decode->op[0].ot;
1203	TCGv carry_in = NULL;
1204	TCGv *carry_out = (cc_op == CC_OP_ADCX ? &decode->cc_dst : &decode->cc_src2);
1205	TCGv zero;
1206
1207	decode->cc_op = cc_op;
1208	*carry_out = tcg_temp_new();
1209	if (CC_OP_HAS_EFLAGS(s->cc_op)) {
1210	decode->cc_src = cpu_cc_src;
1211
1212	/* Re-use the carry-out from a previous round? */
1213	if (s->cc_op == cc_op \|\| s->cc_op == CC_OP_ADCOX) {
1214	carry_in = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2);
1215	}
1216
1217	/* Preserve the opposite carry from previous rounds? */
1218	if (s->cc_op != cc_op && s->cc_op != CC_OP_EFLAGS) {
1219	decode->cc_op = CC_OP_ADCOX;
1220	if (carry_out == &decode->cc_dst) {
1221	decode->cc_src2 = cpu_cc_src2;
1222	} else {
1223	decode->cc_dst = cpu_cc_dst;
1224	}
1225	}
1226	} else {
1227	decode->cc_src = tcg_temp_new();
1228	gen_mov_eflags(s, decode->cc_src);
1229	}
1230
1231	if (!carry_in) {
1232	/* Get carry_in out of EFLAGS. */
1233	carry_in = tcg_temp_new();
1234	tcg_gen_extract_tl(carry_in, decode->cc_src,
1235	ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1);
1236	}
1237
1238	switch (ot) {
1239	case MO_32:
1240	#ifdef TARGET_X86_64
1241	/* If TL is 64-bit just do everything in 64-bit arithmetic. */
1242	tcg_gen_ext32u_tl(s->T0, s->T0);
1243	tcg_gen_ext32u_tl(s->T1, s->T1);
1244	tcg_gen_add_i64(s->T0, s->T0, s->T1);
1245	tcg_gen_add_i64(s->T0, s->T0, carry_in);
1246	tcg_gen_shri_i64(*carry_out, s->T0, 32);
1247	break;
1248
1249	case MO_64:
1250	#endif
1251	zero = tcg_constant_tl(0);
1252	tcg_gen_add2_tl(s->T0, *carry_out, s->T0, zero, carry_in, zero);
1253	tcg_gen_add2_tl(s->T0, carry_out, s->T0, carry_out, s->T1, zero);
1254	break;
1255
1256	default:
1257	g_assert_not_reached();
1258	}
1259	}
1260
1261	static void gen_ADCX(DisasContext s, X86DecodedInsn decode)
1262	{
1263	gen_ADCOX(s, decode, CC_OP_ADCX);
1264	}
1265
1266	static void gen_ADD(DisasContext s, X86DecodedInsn decode)
1267	{
1268	MemOp ot = decode->op[1].ot;
1269
1270	if (s->prefix & PREFIX_LOCK) {
1271	tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1,
1272	s->mem_index, ot \| MO_LE);
1273	} else {
1274	tcg_gen_add_tl(s->T0, s->T0, s->T1);
1275	}
1276	prepare_update2_cc(decode, s, CC_OP_ADDB + ot);
1277	}
1278
1279	static void gen_ADOX(DisasContext s, X86DecodedInsn decode)
1280	{
1281	gen_ADCOX(s, decode, CC_OP_ADOX);
1282	}
1283
1284	static void gen_AND(DisasContext s, X86DecodedInsn decode)
1285	{
1286	MemOp ot = decode->op[1].ot;
1287
1288	if (s->prefix & PREFIX_LOCK) {
1289	tcg_gen_atomic_and_fetch_tl(s->T0, s->A0, s->T1,
1290	s->mem_index, ot \| MO_LE);
1291	} else {
1292	tcg_gen_and_tl(s->T0, s->T0, s->T1);
1293	}
1294	prepare_update1_cc(decode, s, CC_OP_LOGICB + ot);
1295	}
1296
1297	static void gen_ANDN(DisasContext s, X86DecodedInsn decode)
1298	{
1299	MemOp ot = decode->op[0].ot;
1300
1301	tcg_gen_andc_tl(s->T0, s->T1, s->T0);
1302	prepare_update1_cc(decode, s, CC_OP_LOGICB + ot);
1303	}
1304
1305	static void gen_ARPL(DisasContext s, X86DecodedInsn decode)
1306	{
1307	TCGv zf = tcg_temp_new();
1308	TCGv flags = tcg_temp_new();
1309
1310	gen_mov_eflags(s, flags);
1311
1312	/* Compute adjusted DST in T1, merging in SRC[RPL]. */
1313	tcg_gen_deposit_tl(s->T1, s->T0, s->T1, 0, 2);
1314
1315	/* Z flag set if DST[RPL] < SRC[RPL] */
1316	tcg_gen_setcond_tl(TCG_COND_LTU, zf, s->T0, s->T1);
1317	tcg_gen_deposit_tl(flags, flags, zf, ctz32(CC_Z), 1);
1318
1319	/* Place maximum RPL in DST */
1320	tcg_gen_umax_tl(s->T0, s->T0, s->T1);
1321
1322	decode->cc_src = flags;
1323	decode->cc_op = CC_OP_EFLAGS;
1324	}
1325
1326	static void gen_BEXTR(DisasContext s, X86DecodedInsn decode)
1327	{
1328	MemOp ot = decode->op[0].ot;
1329	TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31);
1330	TCGv zero = tcg_constant_tl(0);
1331	TCGv mone = tcg_constant_tl(-1);
1332
1333	/*
1334	* Extract START, and shift the operand.
1335	* Shifts larger than operand size get zeros.
1336	*/
1337	tcg_gen_ext8u_tl(s->A0, s->T1);
1338	tcg_gen_shr_tl(s->T0, s->T0, s->A0);
1339
1340	tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero);
1341
1342	/*
1343	* Extract the LEN into an inverse mask. Lengths larger than
1344	* operand size get all zeros, length 0 gets all ones.
1345	*/
1346	tcg_gen_extract_tl(s->A0, s->T1, 8, 8);
1347	tcg_gen_shl_tl(s->T1, mone, s->A0);
1348	tcg_gen_movcond_tl(TCG_COND_LEU, s->T1, s->A0, bound, s->T1, zero);
1349	tcg_gen_andc_tl(s->T0, s->T0, s->T1);
1350
1351	prepare_update1_cc(decode, s, CC_OP_LOGICB + ot);
1352	}
1353
1354	static void gen_BLSI(DisasContext s, X86DecodedInsn decode)
1355	{
1356	MemOp ot = decode->op[0].ot;
1357
1358	/* input in T1, which is ready for prepare_update2_cc */
1359	tcg_gen_neg_tl(s->T0, s->T1);
1360	tcg_gen_and_tl(s->T0, s->T0, s->T1);
1361	prepare_update2_cc(decode, s, CC_OP_BLSIB + ot);
1362	}
1363
1364	static void gen_BLSMSK(DisasContext s, X86DecodedInsn decode)
1365	{
1366	MemOp ot = decode->op[0].ot;
1367
1368	/* input in T1, which is ready for prepare_update2_cc */
1369	tcg_gen_subi_tl(s->T0, s->T1, 1);
1370	tcg_gen_xor_tl(s->T0, s->T0, s->T1);
1371	prepare_update2_cc(decode, s, CC_OP_BMILGB + ot);
1372	}
1373
1374	static void gen_BLSR(DisasContext s, X86DecodedInsn decode)
1375	{
1376	MemOp ot = decode->op[0].ot;
1377
1378	/* input in T1, which is ready for prepare_update2_cc */
1379	tcg_gen_subi_tl(s->T0, s->T1, 1);
1380	tcg_gen_and_tl(s->T0, s->T0, s->T1);
1381	prepare_update2_cc(decode, s, CC_OP_BMILGB + ot);
1382	}
1383
1384	static void gen_BOUND(DisasContext s, X86DecodedInsn decode)
1385	{
1386	TCGv_i32 op = tcg_temp_new_i32();
1387	tcg_gen_trunc_tl_i32(op, s->T0);
1388	if (decode->op[1].ot == MO_16) {
1389	gen_helper_boundw(tcg_env, s->A0, op);
1390	} else {
1391	gen_helper_boundl(tcg_env, s->A0, op);
1392	}
1393	}
1394
1395	/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */
1396	static void gen_BSF(DisasContext s, X86DecodedInsn decode)
1397	{
1398	MemOp ot = decode->op[0].ot;
1399
1400	/* Only the Z bit is defined and it is related to the input. */
1401	decode->cc_dst = tcg_temp_new();
1402	decode->cc_op = CC_OP_LOGICB + ot;
1403	tcg_gen_mov_tl(decode->cc_dst, s->T0);
1404
1405	/*
1406	* The manual says that the output is undefined when the
1407	* input is zero, but real hardware leaves it unchanged, and
1408	* real programs appear to depend on that. Accomplish this
1409	* by passing the output as the value to return upon zero.
1410	*/
1411	tcg_gen_ctz_tl(s->T0, s->T0, s->T1);
1412	}
1413
1414	/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */
1415	static void gen_BSR(DisasContext s, X86DecodedInsn decode)
1416	{
1417	MemOp ot = decode->op[0].ot;
1418
1419	/* Only the Z bit is defined and it is related to the input. */
1420	decode->cc_dst = tcg_temp_new();
1421	decode->cc_op = CC_OP_LOGICB + ot;
1422	tcg_gen_mov_tl(decode->cc_dst, s->T0);
1423
1424	/*
1425	* The manual says that the output is undefined when the
1426	* input is zero, but real hardware leaves it unchanged, and
1427	* real programs appear to depend on that. Accomplish this
1428	* by passing the output as the value to return upon zero.
1429	* Plus, return the bit index of the first 1 bit.
1430	*/
1431	tcg_gen_xori_tl(s->T1, s->T1, TARGET_LONG_BITS - 1);
1432	tcg_gen_clz_tl(s->T0, s->T0, s->T1);
1433	tcg_gen_xori_tl(s->T0, s->T0, TARGET_LONG_BITS - 1);
1434	}
1435
1436	static void gen_BSWAP(DisasContext s, X86DecodedInsn decode)
1437	{
1438	#ifdef TARGET_X86_64
1439	if (s->dflag == MO_64) {
1440	tcg_gen_bswap64_i64(s->T0, s->T0);
1441	return;
1442	}
1443	#endif
1444	tcg_gen_bswap32_tl(s->T0, s->T0, TCG_BSWAP_OZ);
1445	}
1446
1447	static TCGv gen_bt_mask(DisasContext s, X86DecodedInsn decode)
1448	{
1449	MemOp ot = decode->op[1].ot;
1450	TCGv mask = tcg_temp_new();
1451
1452	tcg_gen_andi_tl(s->T1, s->T1, (8 << ot) - 1);
1453	tcg_gen_shl_tl(mask, tcg_constant_tl(1), s->T1);
1454	return mask;
1455	}
1456
1457	/* Expects truncated bit index in COUNT, 1 << COUNT in MASK. */
1458	static void gen_bt_flags(DisasContext s, X86DecodedInsn decode, TCGv src,
1459	TCGv count, TCGv mask)
1460	{
1461	TCGv cf;
1462
1463	/*
1464	* C is the result of the test, Z is unchanged, and the others
1465	* are all undefined.
1466	*/
1467	if (s->cc_op == CC_OP_DYNAMIC \|\| CC_OP_HAS_EFLAGS(s->cc_op)) {
1468	/* Generate EFLAGS and replace the C bit. */
1469	cf = tcg_temp_new();
1470	tcg_gen_setcond_tl(TCG_COND_TSTNE, cf, src, mask);
1471	prepare_update_cf(decode, s, cf);
1472	} else {
1473	/*
1474	* Z was going to be computed from the non-zero status of CC_DST.
1475	* We can get that same Z value (and the new C value) by leaving
1476	* CC_DST alone, setting CC_SRC, and using a CC_OP_SAR of the
1477	* same width.
1478	*/
1479	decode->cc_src = tcg_temp_new();
1480	decode->cc_dst = cpu_cc_dst;
1481	decode->cc_op = CC_OP_SARB + cc_op_size(s->cc_op);
1482	tcg_gen_shr_tl(decode->cc_src, src, count);
1483	}
1484	}
1485
1486	static void gen_BT(DisasContext s, X86DecodedInsn decode)
1487	{
1488	TCGv count = s->T1;
1489	TCGv mask;
1490
1491	/*
1492	* Try to ensure that the rhs of the TSTNE condition is a constant (and a
1493	* power of two), as that is more readily available on most TCG backends.
1494	*
1495	* For immediate bit number gen_bt_mask()'s output is already a constant;
1496	* for register bit number, shift the source right and check bit 0.
1497	*/
1498	if (decode->e.op2 == X86_TYPE_I) {
1499	mask = gen_bt_mask(s, decode);
1500	} else {
1501	MemOp ot = decode->op[1].ot;
1502
1503	tcg_gen_andi_tl(s->T1, s->T1, (8 << ot) - 1);
1504	tcg_gen_shr_tl(s->T0, s->T0, s->T1);
1505
1506	count = tcg_constant_tl(0);
1507	mask = tcg_constant_tl(1);
1508	}
1509	gen_bt_flags(s, decode, s->T0, count, mask);
1510	}
1511
1512	static void gen_BTC(DisasContext s, X86DecodedInsn decode)
1513	{
1514	MemOp ot = decode->op[0].ot;
1515	TCGv old = tcg_temp_new();
1516	TCGv mask = gen_bt_mask(s, decode);
1517
1518	if (s->prefix & PREFIX_LOCK) {
1519	tcg_gen_atomic_fetch_xor_tl(old, s->A0, mask, s->mem_index, ot \| MO_LE);
1520	} else {
1521	tcg_gen_mov_tl(old, s->T0);
1522	tcg_gen_xor_tl(s->T0, s->T0, mask);
1523	}
1524
1525	gen_bt_flags(s, decode, old, s->T1, mask);
1526	}
1527
1528	static void gen_BTR(DisasContext s, X86DecodedInsn decode)
1529	{
1530	MemOp ot = decode->op[0].ot;
1531	TCGv old = tcg_temp_new();
1532	TCGv mask = gen_bt_mask(s, decode);
1533
1534	if (s->prefix & PREFIX_LOCK) {
1535	TCGv maskc = tcg_temp_new();
1536	tcg_gen_not_tl(maskc, mask);
1537	tcg_gen_atomic_fetch_and_tl(old, s->A0, maskc, s->mem_index, ot \| MO_LE);
1538	} else {
1539	tcg_gen_mov_tl(old, s->T0);
1540	tcg_gen_andc_tl(s->T0, s->T0, mask);
1541	}
1542
1543	gen_bt_flags(s, decode, old, s->T1, mask);
1544	}
1545
1546	static void gen_BTS(DisasContext s, X86DecodedInsn decode)
1547	{
1548	MemOp ot = decode->op[0].ot;
1549	TCGv old = tcg_temp_new();
1550	TCGv mask = gen_bt_mask(s, decode);
1551
1552	if (s->prefix & PREFIX_LOCK) {
1553	tcg_gen_atomic_fetch_or_tl(old, s->A0, mask, s->mem_index, ot \| MO_LE);
1554	} else {
1555	tcg_gen_mov_tl(old, s->T0);
1556	tcg_gen_or_tl(s->T0, s->T0, mask);
1557	}
1558
1559	gen_bt_flags(s, decode, old, s->T1, mask);
1560	}
1561
1562	static void gen_BZHI(DisasContext s, X86DecodedInsn decode)
1563	{
1564	MemOp ot = decode->op[0].ot;
1565	TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31);
1566	TCGv zero = tcg_constant_tl(0);
1567	TCGv mone = tcg_constant_tl(-1);
1568
1569	tcg_gen_ext8u_tl(s->T1, s->T1);
1570
1571	tcg_gen_shl_tl(s->A0, mone, s->T1);
1572	tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->T1, bound, s->A0, zero);
1573	tcg_gen_andc_tl(s->T0, s->T0, s->A0);
1574	/*
1575	* Note that since we're using BMILG (in order to get O
1576	* cleared) we need to store the inverse into C.
1577	*/
1578	tcg_gen_setcond_tl(TCG_COND_LEU, s->T1, s->T1, bound);
1579	prepare_update2_cc(decode, s, CC_OP_BMILGB + ot);
1580	}
1581
1582	static void gen_CALL(DisasContext s, X86DecodedInsn decode)
1583	{
1584	gen_push_v(s, eip_next_tl(s));
1585	gen_JMP(s, decode);
1586	}
1587
1588	static void gen_CALL_m(DisasContext s, X86DecodedInsn decode)
1589	{
1590	gen_push_v(s, eip_next_tl(s));
1591	gen_JMP_m(s, decode);
1592	}
1593
1594	static void gen_CALLF(DisasContext s, X86DecodedInsn decode)
1595	{
1596	gen_far_call(s);
1597	}
1598
1599	static void gen_CALLF_m(DisasContext s, X86DecodedInsn decode)
1600	{
1601	MemOp ot = decode->op[1].ot;
1602
1603	gen_op_ld_v(s, ot, s->T0, s->A0);
1604	gen_add_A0_im(s, 1 << ot);
1605	gen_op_ld_v(s, MO_16, s->T1, s->A0);
1606	gen_far_call(s);
1607	}
1608
1609	static void gen_CBW(DisasContext s, X86DecodedInsn decode)
1610	{
1611	MemOp src_ot = decode->op[0].ot - 1;
1612
1613	tcg_gen_ext_tl(s->T0, s->T0, src_ot \| MO_SIGN);
1614	}
1615
1616	static void gen_CLC(DisasContext s, X86DecodedInsn decode)
1617	{
1618	gen_compute_eflags(s);
1619	tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_C);
1620	}
1621
1622	static void gen_CLD(DisasContext s, X86DecodedInsn decode)
1623	{
1624	tcg_gen_st_i32(tcg_constant_i32(1), tcg_env, offsetof(CPUX86State, df));
1625	}
1626
1627	static void gen_CLI(DisasContext s, X86DecodedInsn decode)
1628	{
1629	gen_reset_eflags(s, IF_MASK);
1630	}
1631
1632	static void gen_CLTS(DisasContext s, X86DecodedInsn decode)
1633	{
1634	gen_helper_clts(tcg_env);
1635	/* abort block because static cpu state changed */
1636	s->base.is_jmp = DISAS_EOB_NEXT;
1637	}
1638
1639	static void gen_CMC(DisasContext s, X86DecodedInsn decode)
1640	{
1641	gen_compute_eflags(s);
1642	tcg_gen_xori_tl(cpu_cc_src, cpu_cc_src, CC_C);
1643	}
1644
1645	static void gen_CMOVcc(DisasContext s, X86DecodedInsn decode)
1646	{
1647	gen_cmovcc(s, decode->b & 0xf, s->T0, s->T1);
1648	}
1649
1650	static void gen_CMPccXADD(DisasContext s, X86DecodedInsn decode)
1651	{
1652	TCGLabel *label_top = gen_new_label();
1653	TCGLabel *label_bottom = gen_new_label();
1654	TCGv oldv = tcg_temp_new();
1655	TCGv newv = tcg_temp_new();
1656	TCGv cmpv = tcg_temp_new();
1657	TCGCond cond;
1658
1659	TCGv cmp_lhs, cmp_rhs;
1660	MemOp ot, ot_full;
1661
1662	int jcc_op = (decode->b >> 1) & 7;
1663	static const TCGCond cond_table[8] = {
1664	[JCC_O] = TCG_COND_LT, /* test sign bit by comparing against 0 */
1665	[JCC_B] = TCG_COND_LTU,
1666	[JCC_Z] = TCG_COND_EQ,
1667	[JCC_BE] = TCG_COND_LEU,
1668	[JCC_S] = TCG_COND_LT, /* test sign bit by comparing against 0 */
1669	[JCC_P] = TCG_COND_TSTEQ, /* even parity - tests low bit of popcount */
1670	[JCC_L] = TCG_COND_LT,
1671	[JCC_LE] = TCG_COND_LE,
1672	};
1673
1674	cond = cond_table[jcc_op];
1675	if (decode->b & 1) {
1676	cond = tcg_invert_cond(cond);
1677	}
1678
1679	ot = decode->op[0].ot;
1680	ot_full = ot \| MO_LE;
1681	if (jcc_op >= JCC_S) {
1682	/*
1683	* Sign-extend values before subtracting for S, P (zero/sign extension
1684	* does not matter there) L, LE and their inverses.
1685	*/
1686	ot_full \|= MO_SIGN;
1687	}
1688
1689	tcg_gen_ext_tl(cmpv, cpu_regs[decode->op[1].n], ot_full);
1690
1691	/*
1692	* Cmpxchg loop starts here.
1693	* - s->T1: addition operand (from decoder)
1694	* - s->A0: dest address (from decoder)
1695	* - s->cc_srcT: memory operand (lhs for comparison)
1696	* - cmpv: rhs for comparison
1697	*/
1698	s->cc_srcT = tcg_temp_new();
1699	gen_set_label(label_top);
1700	gen_op_ld_v(s, ot_full, s->cc_srcT, s->A0);
1701	tcg_gen_sub_tl(s->T0, s->cc_srcT, cmpv);
1702
1703	/* Compute the comparison result by hand, to avoid clobbering cc_. /
1704	switch (jcc_op) {
1705	case JCC_O:
1706	/* (src1 ^ src2) & (src1 ^ dst). newv is only used here for a moment */
1707	cmp_lhs = tcg_temp_new(), cmp_rhs = tcg_constant_tl(0);
1708	tcg_gen_xor_tl(newv, s->cc_srcT, s->T0);
1709	tcg_gen_xor_tl(cmp_lhs, s->cc_srcT, cmpv);
1710	tcg_gen_and_tl(cmp_lhs, cmp_lhs, newv);
1711	tcg_gen_sextract_tl(cmp_lhs, cmp_lhs, 0, 8 << ot);
1712	break;
1713
1714	case JCC_P:
1715	cmp_lhs = tcg_temp_new(), cmp_rhs = tcg_constant_tl(1);
1716	tcg_gen_ext8u_tl(cmp_lhs, s->T0);
1717	tcg_gen_ctpop_tl(cmp_lhs, cmp_lhs);
1718	break;
1719
1720	case JCC_S:
1721	cmp_lhs = tcg_temp_new(), cmp_rhs = tcg_constant_tl(0);
1722	tcg_gen_sextract_tl(cmp_lhs, s->T0, 0, 8 << ot);
1723	break;
1724
1725	default:
1726	cmp_lhs = s->cc_srcT, cmp_rhs = cmpv;
1727	break;
1728	}
1729
1730	/* Compute new value: if condition does not hold, just store back s->cc_srcT */
1731	tcg_gen_add_tl(newv, s->cc_srcT, s->T1);
1732	tcg_gen_movcond_tl(cond, newv, cmp_lhs, cmp_rhs, newv, s->cc_srcT);
1733	tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, s->cc_srcT, newv, s->mem_index, ot_full);
1734
1735	/* Exit unconditionally if cmpxchg succeeded. */
1736	tcg_gen_brcond_tl(TCG_COND_EQ, oldv, s->cc_srcT, label_bottom);
1737
1738	/* Try again if there was actually a store to make. */
1739	tcg_gen_brcond_tl(cond, cmp_lhs, cmp_rhs, label_top);
1740	gen_set_label(label_bottom);
1741
1742	/* Store old value to registers only after a successful store. */
1743	gen_writeback(s, decode, 1, s->cc_srcT);
1744
1745	decode->cc_dst = s->T0;
1746	decode->cc_src = cmpv;
1747	decode->cc_op = CC_OP_SUBB + ot;
1748	}
1749
1750	static void gen_CMPS(DisasContext s, X86DecodedInsn decode)
1751	{
1752	MemOp ot = decode->op[2].ot;
1753	gen_repz_nz(s, ot, gen_cmps);
1754	}
1755
1756	static void gen_CMPXCHG(DisasContext s, X86DecodedInsn decode)
1757	{
1758	MemOp ot = decode->op[2].ot;
1759	TCGv cmpv = tcg_temp_new();
1760	TCGv oldv = tcg_temp_new();
1761	TCGv newv = tcg_temp_new();
1762	TCGv dest;
1763
1764	tcg_gen_ext_tl(cmpv, cpu_regs[R_EAX], ot);
1765	tcg_gen_ext_tl(newv, s->T1, ot);
1766	if (s->prefix & PREFIX_LOCK) {
1767	tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, cmpv, newv,
1768	s->mem_index, ot \| MO_LE);
1769	} else {
1770	tcg_gen_ext_tl(oldv, s->T0, ot);
1771	if (decode->op[0].has_ea) {
1772	/*
1773	* Perform an unconditional store cycle like physical cpu;
1774	* must be before changing accumulator to ensure
1775	* idempotency if the store faults and the instruction
1776	* is restarted
1777	*/
1778	tcg_gen_movcond_tl(TCG_COND_EQ, newv, oldv, cmpv, newv, oldv);
1779	gen_op_st_v(s, ot, newv, s->A0);
1780	} else {
1781	/*
1782	* Unlike the memory case, where "the destination operand receives
1783	* a write cycle without regard to the result of the comparison",
1784	* rm must not be touched altogether if the write fails, including
1785	* not zero-extending it on 64-bit processors. So, precompute
1786	* the result of a successful writeback and perform the movcond
1787	* directly on cpu_regs. In case rm is part of RAX, note that this
1788	* movcond and the one below are mutually exclusive is executed.
1789	*/
1790	dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, newv, newv);
1791	tcg_gen_movcond_tl(TCG_COND_EQ, dest, oldv, cmpv, newv, dest);
1792	}
1793	decode->op[0].unit = X86_OP_SKIP;
1794	}
1795
1796	/* Write RAX only if the cmpxchg fails. */
1797	dest = gen_op_deposit_reg_v(s, ot, R_EAX, s->T0, oldv);
1798	tcg_gen_movcond_tl(TCG_COND_NE, dest, oldv, cmpv, s->T0, dest);
1799
1800	s->cc_srcT = cmpv;
1801	decode->cc_dst = tcg_temp_new();
1802	decode->cc_src = oldv;
1803	decode->cc_op = CC_OP_SUBB + ot;
1804
1805	tcg_gen_sub_tl(decode->cc_dst, cmpv, oldv);
1806	}
1807
1808	static void gen_CMPXCHG16B(DisasContext s, X86DecodedInsn decode)
1809	{
1810	#ifdef TARGET_X86_64
1811	MemOp mop = MO_LE \| MO_128 \| MO_ALIGN;
1812	TCGv_i64 t0, t1;
1813	TCGv_i128 cmp, val;
1814
1815	cmp = tcg_temp_new_i128();
1816	val = tcg_temp_new_i128();
1817	tcg_gen_concat_i64_i128(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]);
1818	tcg_gen_concat_i64_i128(val, cpu_regs[R_EBX], cpu_regs[R_ECX]);
1819
1820	/* Only require atomic with LOCK; non-parallel handled in generator. */
1821	if (s->prefix & PREFIX_LOCK) {
1822	tcg_gen_atomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop);
1823	} else {
1824	tcg_gen_nonatomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop);
1825	}
1826
1827	tcg_gen_extr_i128_i64(s->T0, s->T1, val);
1828
1829	/* Determine success after the fact. */
1830	t0 = tcg_temp_new_i64();
1831	t1 = tcg_temp_new_i64();
1832	tcg_gen_xor_i64(t0, s->T0, cpu_regs[R_EAX]);
1833	tcg_gen_xor_i64(t1, s->T1, cpu_regs[R_EDX]);
1834	tcg_gen_or_i64(t0, t0, t1);
1835
1836	/* Update Z. */
1837	gen_compute_eflags(s);
1838	tcg_gen_setcondi_i64(TCG_COND_EQ, t0, t0, 0);
1839	tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, t0, ctz32(CC_Z), 1);
1840
1841	/*
1842	* Extract the result values for the register pair. We may do this
1843	* unconditionally, because on success (Z=1), the old value matches
1844	* the previous value in RDX:RAX.
1845	*/
1846	tcg_gen_mov_i64(cpu_regs[R_EAX], s->T0);
1847	tcg_gen_mov_i64(cpu_regs[R_EDX], s->T1);
1848	#else
1849	abort();
1850	#endif
1851	}
1852
1853	static void gen_CMPXCHG8B(DisasContext s, X86DecodedInsn decode)
1854	{
1855	TCGv_i64 cmp, val, old;
1856	TCGv Z;
1857
1858	cmp = tcg_temp_new_i64();
1859	val = tcg_temp_new_i64();
1860	old = tcg_temp_new_i64();
1861
1862	/* Construct the comparison values from the register pair. */
1863	tcg_gen_concat_tl_i64(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]);
1864	tcg_gen_concat_tl_i64(val, cpu_regs[R_EBX], cpu_regs[R_ECX]);
1865
1866	/* Only require atomic with LOCK; non-parallel handled in generator. */
1867	if (s->prefix & PREFIX_LOCK) {
1868	tcg_gen_atomic_cmpxchg_i64(old, s->A0, cmp, val, s->mem_index, MO_LEUQ);
1869	} else {
1870	tcg_gen_nonatomic_cmpxchg_i64(old, s->A0, cmp, val,
1871	s->mem_index, MO_LEUQ);
1872	}
1873
1874	/* Compute the required value of Z. */
1875	tcg_gen_setcond_i64(TCG_COND_EQ, cmp, old, cmp);
1876	Z = tcg_temp_new();
1877	tcg_gen_trunc_i64_tl(Z, cmp);
1878
1879	/*
1880	* Extract the result values for the register pair.
1881	* For 32-bit, we may do this unconditionally, because on success (Z=1),
1882	* the old value matches the previous value in EDX:EAX. For x86_64,
1883	* the store must be conditional, because we must leave the source
1884	* registers unchanged on success, and zero-extend the writeback
1885	* on failure (Z=0).
1886	*/
1887	if (TARGET_LONG_BITS == 32) {
1888	tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], old);
1889	} else {
1890	TCGv zero = tcg_constant_tl(0);
1891
1892	tcg_gen_extr_i64_tl(s->T0, s->T1, old);
1893	tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EAX], Z, zero,
1894	s->T0, cpu_regs[R_EAX]);
1895	tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EDX], Z, zero,
1896	s->T1, cpu_regs[R_EDX]);
1897	}
1898
1899	/* Update Z. */
1900	gen_compute_eflags(s);
1901	tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, Z, ctz32(CC_Z), 1);
1902	}
1903
1904	static void gen_CPUID(DisasContext s, X86DecodedInsn decode)
1905	{
1906	gen_update_cc_op(s);
1907	gen_update_eip_cur(s);
1908	gen_helper_cpuid(tcg_env);
1909	}
1910
1911	static void gen_CRC32(DisasContext s, X86DecodedInsn decode)
1912	{
1913	MemOp ot = decode->op[2].ot;
1914	TCGv_i32 tmp = tcg_temp_new_i32();
1915
1916	tcg_gen_trunc_tl_i32(tmp, s->T0);
1917	gen_helper_crc32(s->T0, tmp, s->T1, tcg_constant_i32(8 << ot));
1918	}
1919
1920	static void gen_CVTPI2Px(DisasContext s, X86DecodedInsn decode)
1921	{
1922	gen_helper_enter_mmx(tcg_env);
1923	if (s->prefix & PREFIX_DATA) {
1924	gen_helper_cvtpi2pd(tcg_env, OP_PTR0, OP_PTR2);
1925	} else {
1926	gen_helper_cvtpi2ps(tcg_env, OP_PTR0, OP_PTR2);
1927	}
1928	}
1929
1930	static void gen_CVTPx2PI(DisasContext s, X86DecodedInsn decode)
1931	{
1932	gen_helper_enter_mmx(tcg_env);
1933	if (s->prefix & PREFIX_DATA) {
1934	gen_helper_cvtpd2pi(tcg_env, OP_PTR0, OP_PTR2);
1935	} else {
1936	gen_helper_cvtps2pi(tcg_env, OP_PTR0, OP_PTR2);
1937	}
1938	}
1939
1940	static void gen_CVTTPx2PI(DisasContext s, X86DecodedInsn decode)
1941	{
1942	gen_helper_enter_mmx(tcg_env);
1943	if (s->prefix & PREFIX_DATA) {
1944	gen_helper_cvttpd2pi(tcg_env, OP_PTR0, OP_PTR2);
1945	} else {
1946	gen_helper_cvttps2pi(tcg_env, OP_PTR0, OP_PTR2);
1947	}
1948	}
1949
1950	static void gen_CWD(DisasContext s, X86DecodedInsn decode)
1951	{
1952	int shift = 8 << decode->op[0].ot;
1953
1954	tcg_gen_sextract_tl(s->T0, s->T0, shift - 1, 1);
1955	}
1956
1957	static void gen_DAA(DisasContext s, X86DecodedInsn decode)
1958	{
1959	gen_update_cc_op(s);
1960	gen_helper_daa(tcg_env);
1961	assume_cc_op(s, CC_OP_EFLAGS);
1962	}
1963
1964	static void gen_DAS(DisasContext s, X86DecodedInsn decode)
1965	{
1966	gen_update_cc_op(s);
1967	gen_helper_das(tcg_env);
1968	assume_cc_op(s, CC_OP_EFLAGS);
1969	}
1970
1971	static void gen_DEC(DisasContext s, X86DecodedInsn decode)
1972	{
1973	MemOp ot = decode->op[1].ot;
1974
1975	tcg_gen_movi_tl(s->T1, -1);
1976	if (s->prefix & PREFIX_LOCK) {
1977	tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1,
1978	s->mem_index, ot \| MO_LE);
1979	} else {
1980	tcg_gen_add_tl(s->T0, s->T0, s->T1);
1981	}
1982	prepare_update_cc_incdec(decode, s, CC_OP_DECB + ot);
1983	}
1984
1985	static void gen_DIV(DisasContext s, X86DecodedInsn decode)
1986	{
1987	MemOp ot = decode->op[1].ot;
1988
1989	switch(ot) {
1990	case MO_8:
1991	gen_helper_divb_AL(tcg_env, s->T0);
1992	break;
1993	case MO_16:
1994	gen_helper_divw_AX(tcg_env, s->T0);
1995	break;
1996	case MO_32:
1997	gen_helper_divl_EAX(tcg_env, s->T0);
1998	break;
1999	#ifdef TARGET_X86_64
2000	case MO_64:
2001	gen_helper_divq_EAX(tcg_env, s->T0);
2002	break;
2003	#endif
2004	default:
2005	g_assert_not_reached();
2006	}
2007	}
2008
2009	static void gen_EMMS(DisasContext s, X86DecodedInsn decode)
2010	{
2011	gen_helper_emms(tcg_env);
2012	}
2013
2014	static void gen_ENTER(DisasContext s, X86DecodedInsn decode)
2015	{
2016	gen_enter(s, decode->op[1].imm, decode->op[2].imm);
2017	}
2018
2019	static void gen_EXTRQ_i(DisasContext s, X86DecodedInsn decode)
2020	{
2021	TCGv_i32 length = tcg_constant_i32(decode->immediate & 63);
2022	TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63);
2023
2024	gen_helper_extrq_i(tcg_env, OP_PTR0, index, length);
2025	}
2026
2027	static void gen_EXTRQ_r(DisasContext s, X86DecodedInsn decode)
2028	{
2029	gen_helper_extrq_r(tcg_env, OP_PTR0, OP_PTR2);
2030	}
2031
2032	static void gen_FXRSTOR(DisasContext s, X86DecodedInsn decode)
2033	{
2034	if ((s->flags & HF_EM_MASK) \|\| (s->flags & HF_TS_MASK)) {
2035	gen_NM_exception(s);
2036	} else {
2037	gen_helper_fxrstor(tcg_env, s->A0);
2038	}
2039	}
2040
2041	static void gen_FXSAVE(DisasContext s, X86DecodedInsn decode)
2042	{
2043	if ((s->flags & HF_EM_MASK) \|\| (s->flags & HF_TS_MASK)) {
2044	gen_NM_exception(s);
2045	} else {
2046	gen_helper_fxsave(tcg_env, s->A0);
2047	}
2048	}
2049
2050	static void gen_HLT(DisasContext s, X86DecodedInsn decode)
2051	{
2052	#ifdef CONFIG_SYSTEM_ONLY
2053	gen_update_cc_op(s);
2054	gen_update_eip_next(s);
2055	gen_helper_hlt(tcg_env);
2056	s->base.is_jmp = DISAS_NORETURN;
2057	#endif
2058	}
2059
2060	static void gen_IDIV(DisasContext s, X86DecodedInsn decode)
2061	{
2062	MemOp ot = decode->op[1].ot;
2063
2064	switch(ot) {
2065	case MO_8:
2066	gen_helper_idivb_AL(tcg_env, s->T0);
2067	break;
2068	case MO_16:
2069	gen_helper_idivw_AX(tcg_env, s->T0);
2070	break;
2071	case MO_32:
2072	gen_helper_idivl_EAX(tcg_env, s->T0);
2073	break;
2074	#ifdef TARGET_X86_64
2075	case MO_64:
2076	gen_helper_idivq_EAX(tcg_env, s->T0);
2077	break;
2078	#endif
2079	default:
2080	g_assert_not_reached();
2081	}
2082	}
2083
2084	static void gen_IMUL3(DisasContext s, X86DecodedInsn decode)
2085	{
2086	MemOp ot = decode->op[0].ot;
2087	TCGv cc_src_rhs;
2088
2089	switch (ot) {
2090	case MO_16:
2091	/* s->T0 already sign-extended */
2092	tcg_gen_ext16s_tl(s->T1, s->T1);
2093	tcg_gen_mul_tl(s->T0, s->T0, s->T1);
2094	/* Compare the full result to the extension of the truncated result. */
2095	tcg_gen_ext16s_tl(s->T1, s->T0);
2096	cc_src_rhs = s->T0;
2097	break;
2098
2099	case MO_32:
2100	#ifdef TARGET_X86_64
2101	/*
2102	* This produces fewer TCG ops, and better code if flags are needed.
2103	* s->T0 is already sign-extended.
2104	*/
2105	tcg_gen_ext32s_tl(s->T1, s->T1);
2106	tcg_gen_mul_tl(s->T0, s->T0, s->T1);
2107	/* Compare the full result to the extension of the truncated result. */
2108	tcg_gen_ext32s_tl(s->T1, s->T0);
2109	cc_src_rhs = s->T0;
2110	break;
2111
2112	case MO_64:
2113	#endif
2114	cc_src_rhs = tcg_temp_new();
2115	tcg_gen_muls2_tl(s->T0, cc_src_rhs, s->T0, s->T1);
2116	/* Compare the high part to the sign bit of the truncated result */
2117	tcg_gen_sari_tl(s->T1, s->T0, TARGET_LONG_BITS - 1);
2118	break;
2119
2120	default:
2121	g_assert_not_reached();
2122	}
2123
2124	tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs);
2125	prepare_update2_cc(decode, s, CC_OP_MULB + ot);
2126	}
2127
2128	static void gen_IMUL(DisasContext s, X86DecodedInsn decode)
2129	{
2130	MemOp ot = decode->op[1].ot;
2131	TCGv cc_src_rhs;
2132
2133	switch (ot) {
2134	case MO_8:
2135	/* s->T0 already sign-extended */
2136	tcg_gen_ext8s_tl(s->T1, s->T1);
2137	tcg_gen_mul_tl(s->T0, s->T0, s->T1);
2138	gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
2139	/* Compare the full result to the extension of the truncated result. */
2140	tcg_gen_ext8s_tl(s->T1, s->T0);
2141	cc_src_rhs = s->T0;
2142	break;
2143
2144	case MO_16:
2145	/* s->T0 already sign-extended */
2146	tcg_gen_ext16s_tl(s->T1, s->T1);
2147	tcg_gen_mul_tl(s->T0, s->T0, s->T1);
2148	gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
2149	tcg_gen_shri_tl(s->T1, s->T0, 16);
2150	gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1);
2151	/* Compare the full result to the extension of the truncated result. */
2152	tcg_gen_ext16s_tl(s->T1, s->T0);
2153	cc_src_rhs = s->T0;
2154	break;
2155
2156	case MO_32:
2157	#ifdef TARGET_X86_64
2158	/* s->T0 already sign-extended */
2159	tcg_gen_ext32s_tl(s->T1, s->T1);
2160	tcg_gen_mul_tl(s->T0, s->T0, s->T1);
2161	tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0);
2162	tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32);
2163	/* Compare the full result to the extension of the truncated result. */
2164	tcg_gen_ext32s_tl(s->T1, s->T0);
2165	cc_src_rhs = s->T0;
2166	break;
2167
2168	case MO_64:
2169	#endif
2170	tcg_gen_muls2_tl(s->T0, cpu_regs[R_EDX], s->T0, s->T1);
2171	tcg_gen_mov_tl(cpu_regs[R_EAX], s->T0);
2172
2173	/* Compare the high part to the sign bit of the truncated result */
2174	tcg_gen_negsetcondi_tl(TCG_COND_LT, s->T1, s->T0, 0);
2175	cc_src_rhs = cpu_regs[R_EDX];
2176	break;
2177
2178	default:
2179	g_assert_not_reached();
2180	}
2181
2182	tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs);
2183	prepare_update2_cc(decode, s, CC_OP_MULB + ot);
2184	}
2185
2186	static void gen_IN(DisasContext s, X86DecodedInsn decode)
2187	{
2188	MemOp ot = decode->op[0].ot;
2189	TCGv_i32 port = tcg_temp_new_i32();
2190
2191	tcg_gen_trunc_tl_i32(port, s->T0);
2192	tcg_gen_ext16u_i32(port, port);
2193	if (!gen_check_io(s, ot, port, SVM_IOIO_TYPE_MASK)) {
2194	return;
2195	}
2196	translator_io_start(&s->base);
2197	gen_helper_in_func(ot, s->T0, port);
2198	gen_writeback(s, decode, 0, s->T0);
2199	gen_bpt_io(s, port, ot);
2200	}
2201
2202	static void gen_INC(DisasContext s, X86DecodedInsn decode)
2203	{
2204	MemOp ot = decode->op[1].ot;
2205
2206	tcg_gen_movi_tl(s->T1, 1);
2207	if (s->prefix & PREFIX_LOCK) {
2208	tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1,
2209	s->mem_index, ot \| MO_LE);
2210	} else {
2211	tcg_gen_add_tl(s->T0, s->T0, s->T1);
2212	}
2213	prepare_update_cc_incdec(decode, s, CC_OP_INCB + ot);
2214	}
2215
2216	static void gen_INS(DisasContext s, X86DecodedInsn decode)
2217	{
2218	MemOp ot = decode->op[1].ot;
2219	TCGv_i32 port = tcg_temp_new_i32();
2220
2221	tcg_gen_trunc_tl_i32(port, s->T1);
2222	tcg_gen_ext16u_i32(port, port);
2223	if (!gen_check_io(s, ot, port,
2224	SVM_IOIO_TYPE_MASK \| SVM_IOIO_STR_MASK)) {
2225	return;
2226	}
2227
2228	translator_io_start(&s->base);
2229	gen_repz(s, ot, gen_ins);
2230	}
2231
2232	static void gen_INSERTQ_i(DisasContext s, X86DecodedInsn decode)
2233	{
2234	TCGv_i32 length = tcg_constant_i32(decode->immediate & 63);
2235	TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63);
2236
2237	gen_helper_insertq_i(tcg_env, OP_PTR0, OP_PTR1, index, length);
2238	}
2239
2240	static void gen_INSERTQ_r(DisasContext s, X86DecodedInsn decode)
2241	{
2242	gen_helper_insertq_r(tcg_env, OP_PTR0, OP_PTR2);
2243	}
2244
2245	static void gen_INT(DisasContext s, X86DecodedInsn decode)
2246	{
2247	gen_interrupt(s, decode->immediate);
2248	}
2249
2250	static void gen_INT1(DisasContext s, X86DecodedInsn decode)
2251	{
2252	gen_update_cc_op(s);
2253	gen_update_eip_next(s);
2254	gen_helper_icebp(tcg_env);
2255	s->base.is_jmp = DISAS_NORETURN;
2256	}
2257
2258	static void gen_INT3(DisasContext s, X86DecodedInsn decode)
2259	{
2260	gen_interrupt(s, EXCP03_INT3);
2261	}
2262
2263	static void gen_INTO(DisasContext s, X86DecodedInsn decode)
2264	{
2265	gen_update_cc_op(s);
2266	gen_update_eip_cur(s);
2267	gen_helper_into(tcg_env, cur_insn_len_i32(s));
2268	}
2269
2270	static void gen_IRET(DisasContext s, X86DecodedInsn decode)
2271	{
2272	if (!PE(s) \|\| VM86(s)) {
2273	gen_helper_iret_real(tcg_env, tcg_constant_i32(s->dflag - 1));
2274	} else {
2275	gen_helper_iret_protected(tcg_env, tcg_constant_i32(s->dflag - 1),
2276	eip_next_i32(s));
2277	}
2278	assume_cc_op(s, CC_OP_EFLAGS);
2279	s->base.is_jmp = DISAS_EOB_ONLY;
2280	}
2281
2282	static void gen_Jcc(DisasContext s, X86DecodedInsn decode)
2283	{
2284	TCGLabel *taken = gen_new_label();
2285
2286	gen_bnd_jmp(s);
2287	gen_jcc(s, decode->b & 0xf, taken);
2288	gen_conditional_jump_labels(s, decode->immediate, NULL, taken);
2289	}
2290
2291	static void gen_JCXZ(DisasContext s, X86DecodedInsn decode)
2292	{
2293	TCGLabel *taken = gen_new_label();
2294
2295	gen_update_cc_op(s);
2296	gen_op_jz_ecx(s, taken);
2297	gen_conditional_jump_labels(s, decode->immediate, NULL, taken);
2298	}
2299
2300	static void gen_JMP(DisasContext s, X86DecodedInsn decode)
2301	{
2302	gen_update_cc_op(s);
2303	gen_jmp_rel(s, s->dflag, decode->immediate, 0);
2304	}
2305
2306	static void gen_JMP_m(DisasContext s, X86DecodedInsn decode)
2307	{
2308	gen_op_jmp_v(s, s->T0);
2309	gen_bnd_jmp(s);
2310	s->base.is_jmp = DISAS_JUMP;
2311	}
2312
2313	static void gen_JMPF(DisasContext s, X86DecodedInsn decode)
2314	{
2315	gen_far_jmp(s);
2316	}
2317
2318	static void gen_JMPF_m(DisasContext s, X86DecodedInsn decode)
2319	{
2320	MemOp ot = decode->op[1].ot;
2321
2322	gen_op_ld_v(s, ot, s->T0, s->A0);
2323	gen_add_A0_im(s, 1 << ot);
2324	gen_op_ld_v(s, MO_16, s->T1, s->A0);
2325	gen_far_jmp(s);
2326	}
2327
2328	static void gen_LAHF(DisasContext s, X86DecodedInsn decode)
2329	{
2330	if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) {
2331	return gen_illegal_opcode(s);
2332	}
2333	gen_compute_eflags(s);
2334	/* Note: gen_compute_eflags() only gives the condition codes */
2335	tcg_gen_ori_tl(s->T0, cpu_cc_src, 0x02);
2336	tcg_gen_deposit_tl(cpu_regs[R_EAX], cpu_regs[R_EAX], s->T0, 8, 8);
2337	}
2338
2339	static void gen_LAR(DisasContext s, X86DecodedInsn decode)
2340	{
2341	MemOp ot = decode->op[0].ot;
2342	TCGv result = tcg_temp_new();
2343	TCGv dest;
2344
2345	gen_compute_eflags(s);
2346	gen_update_cc_op(s);
2347	gen_helper_lar(result, tcg_env, s->T0);
2348
2349	/* Perform writeback here to skip it if ZF=0. */
2350	decode->op[0].unit = X86_OP_SKIP;
2351	dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result);
2352	tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z),
2353	result, dest);
2354	}
2355
2356	static void gen_LDMXCSR(DisasContext s, X86DecodedInsn decode)
2357	{
2358	TCGv_i32 tmp = tcg_temp_new_i32();
2359
2360	tcg_gen_trunc_tl_i32(tmp, s->T0);
2361	gen_helper_ldmxcsr(tcg_env, tmp);
2362	}
2363
2364	static void gen_lxx_seg(DisasContext s, X86DecodedInsn decode, int seg)
2365	{
2366	MemOp ot = decode->op[0].ot;
2367
2368	/* Offset already in s->T0. */
2369	gen_add_A0_im(s, 1 << ot);
2370	gen_op_ld_v(s, MO_16, s->T1, s->A0);
2371
2372	/* load the segment here to handle exceptions properly */
2373	gen_movl_seg(s, seg, s->T1, false);
2374	}
2375
2376	static void gen_LDS(DisasContext s, X86DecodedInsn decode)
2377	{
2378	gen_lxx_seg(s, decode, R_DS);
2379	}
2380
2381	static void gen_LEA(DisasContext s, X86DecodedInsn decode)
2382	{
2383	TCGv ea = gen_lea_modrm_1(s, decode->mem, false);
2384	gen_lea_v_seg_dest(s, s->aflag, s->T0, ea, -1, -1);
2385	}
2386
2387	static void gen_LEAVE(DisasContext s, X86DecodedInsn decode)
2388	{
2389	gen_leave(s);
2390	}
2391
2392	static void gen_LES(DisasContext s, X86DecodedInsn decode)
2393	{
2394	gen_lxx_seg(s, decode, R_ES);
2395	}
2396
2397	static void gen_LFENCE(DisasContext s, X86DecodedInsn decode)
2398	{
2399	tcg_gen_mb(TCG_MO_LD_LD \| TCG_BAR_SC);
2400	}
2401
2402	static void gen_LFS(DisasContext s, X86DecodedInsn decode)
2403	{
2404	gen_lxx_seg(s, decode, R_FS);
2405	}
2406
2407	static void gen_LGS(DisasContext s, X86DecodedInsn decode)
2408	{
2409	gen_lxx_seg(s, decode, R_GS);
2410	}
2411
2412	static void gen_LODS(DisasContext s, X86DecodedInsn decode)
2413	{
2414	MemOp ot = decode->op[1].ot;
2415	gen_repz(s, ot, gen_lods);
2416	}
2417
2418	static void gen_LOOP(DisasContext s, X86DecodedInsn decode)
2419	{
2420	TCGLabel *taken = gen_new_label();
2421
2422	gen_update_cc_op(s);
2423	gen_op_add_reg_im(s, s->aflag, R_ECX, -1);
2424	gen_op_jnz_ecx(s, taken);
2425	gen_conditional_jump_labels(s, decode->immediate, NULL, taken);
2426	}
2427
2428	static void gen_LOOPE(DisasContext s, X86DecodedInsn decode)
2429	{
2430	TCGLabel *taken = gen_new_label();
2431	TCGLabel *not_taken = gen_new_label();
2432
2433	gen_update_cc_op(s);
2434	gen_op_add_reg_im(s, s->aflag, R_ECX, -1);
2435	gen_op_jz_ecx(s, not_taken);
2436	gen_jcc(s, (JCC_Z << 1), taken); /* jz taken */
2437	gen_conditional_jump_labels(s, decode->immediate, not_taken, taken);
2438	}
2439
2440	static void gen_LOOPNE(DisasContext s, X86DecodedInsn decode)
2441	{
2442	TCGLabel *taken = gen_new_label();
2443	TCGLabel *not_taken = gen_new_label();
2444
2445	gen_update_cc_op(s);
2446	gen_op_add_reg_im(s, s->aflag, R_ECX, -1);
2447	gen_op_jz_ecx(s, not_taken);
2448	gen_jcc(s, (JCC_Z << 1) \| 1, taken); /* jnz taken */
2449	gen_conditional_jump_labels(s, decode->immediate, not_taken, taken);
2450	}
2451
2452	static void gen_LSL(DisasContext s, X86DecodedInsn decode)
2453	{
2454	MemOp ot = decode->op[0].ot;
2455	TCGv result = tcg_temp_new();
2456	TCGv dest;
2457
2458	gen_compute_eflags(s);
2459	gen_update_cc_op(s);
2460	gen_helper_lsl(result, tcg_env, s->T0);
2461
2462	/* Perform writeback here to skip it if ZF=0. */
2463	decode->op[0].unit = X86_OP_SKIP;
2464	dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result);
2465	tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z),
2466	result, dest);
2467	}
2468
2469	static void gen_LSS(DisasContext s, X86DecodedInsn decode)
2470	{
2471	gen_lxx_seg(s, decode, R_SS);
2472	}
2473
2474	static void gen_LZCNT(DisasContext s, X86DecodedInsn decode)
2475	{
2476	MemOp ot = decode->op[0].ot;
2477
2478	/* C bit (cc_src) is defined related to the input. */
2479	decode->cc_src = tcg_temp_new();
2480	decode->cc_dst = s->T0;
2481	decode->cc_op = CC_OP_BMILGB + ot;
2482	tcg_gen_mov_tl(decode->cc_src, s->T0);
2483
2484	/*
2485	* Reduce the target_ulong result by the number of zeros that
2486	* we expect to find at the top.
2487	*/
2488	tcg_gen_clzi_tl(s->T0, s->T0, TARGET_LONG_BITS);
2489	tcg_gen_subi_tl(s->T0, s->T0, TARGET_LONG_BITS - (8 << ot));
2490	}
2491
2492	static void gen_MFENCE(DisasContext s, X86DecodedInsn decode)
2493	{
2494	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_SC);
2495	}
2496
2497	static void gen_MOV(DisasContext s, X86DecodedInsn decode)
2498	{
2499	/* nothing to do! */
2500	}
2501	#define gen_NOP gen_MOV
2502
2503	static void gen_MASKMOV(DisasContext s, X86DecodedInsn decode)
2504	{
2505	gen_lea_v_seg(s, cpu_regs[R_EDI], R_DS, s->override);
2506
2507	if (s->prefix & PREFIX_DATA) {
2508	gen_helper_maskmov_xmm(tcg_env, OP_PTR1, OP_PTR2, s->A0);
2509	} else {
2510	gen_helper_maskmov_mmx(tcg_env, OP_PTR1, OP_PTR2, s->A0);
2511	}
2512	}
2513
2514	static void gen_MOVBE(DisasContext s, X86DecodedInsn decode)
2515	{
2516	MemOp ot = decode->op[0].ot;
2517
2518	/* M operand type does not load/store */
2519	if (decode->e.op0 == X86_TYPE_M) {
2520	tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot \| MO_BE);
2521	} else {
2522	tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot \| MO_BE);
2523	}
2524	}
2525
2526	static void gen_MOVD_from(DisasContext s, X86DecodedInsn decode)
2527	{
2528	MemOp ot = decode->op[2].ot;
2529
2530	switch (ot) {
2531	case MO_32:
2532	#ifdef TARGET_X86_64
2533	tcg_gen_ld32u_tl(s->T0, tcg_env, decode->op[2].offset);
2534	break;
2535	case MO_64:
2536	#endif
2537	tcg_gen_ld_tl(s->T0, tcg_env, decode->op[2].offset);
2538	break;
2539	default:
2540	abort();
2541	}
2542	}
2543
2544	static void gen_MOVD_to(DisasContext s, X86DecodedInsn decode)
2545	{
2546	MemOp ot = decode->op[2].ot;
2547	int vec_len = vector_len(s, decode);
2548	int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0);
2549
2550	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
2551
2552	switch (ot) {
2553	case MO_32:
2554	#ifdef TARGET_X86_64
2555	tcg_gen_st32_tl(s->T1, tcg_env, lo_ofs);
2556	break;
2557	case MO_64:
2558	#endif
2559	tcg_gen_st_tl(s->T1, tcg_env, lo_ofs);
2560	break;
2561	default:
2562	g_assert_not_reached();
2563	}
2564	}
2565
2566	static void gen_MOVDQ(DisasContext s, X86DecodedInsn decode)
2567	{
2568	gen_store_sse(s, decode, decode->op[2].offset);
2569	}
2570
2571	static void gen_MOVMSK(DisasContext s, X86DecodedInsn decode)
2572	{
2573	typeof(gen_helper_movmskps_ymm) ps, pd, *fn;
2574	TCGv_i32 tmp = tcg_temp_new_i32();
2575
2576	ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm;
2577	pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm;
2578	fn = s->prefix & PREFIX_DATA ? pd : ps;
2579	fn(tmp, tcg_env, OP_PTR2);
2580	tcg_gen_extu_i32_tl(s->T0, tmp);
2581	}
2582
2583	static void gen_MOVQ(DisasContext s, X86DecodedInsn decode)
2584	{
2585	int vec_len = vector_len(s, decode);
2586	int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0);
2587	TCGv_i64 t = tcg_temp_new_i64();
2588
2589	tcg_gen_ld_i64(t, tcg_env, decode->op[2].offset);
2590	if (decode->op[0].has_ea) {
2591	tcg_gen_qemu_st_i64(t, s->A0, s->mem_index, MO_LEUQ);
2592	} else {
2593	/*
2594	* tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would
2595	* seem to work, but it does not on big-endian platforms; the cleared parts
2596	* are always at higher addresses, but cross-endian emulation inverts the
2597	* byte order so that the cleared parts need to be at lower addresses.
2598	* Because oprsz is 8, we see this here even for SSE; but more in general,
2599	* it disqualifies using oprsz < maxsz to emulate VEX128.
2600	*/
2601	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
2602	tcg_gen_st_i64(t, tcg_env, lo_ofs);
2603	}
2604	}
2605
2606	static void gen_MOVq_dq(DisasContext s, X86DecodedInsn decode)
2607	{
2608	gen_helper_enter_mmx(tcg_env);
2609	/* Otherwise the same as any other movq. */
2610	return gen_MOVQ(s, decode);
2611	}
2612
2613	static void gen_MOVS(DisasContext s, X86DecodedInsn decode)
2614	{
2615	MemOp ot = decode->op[2].ot;
2616	gen_repz(s, ot, gen_movs);
2617	}
2618
2619	static void gen_MUL(DisasContext s, X86DecodedInsn decode)
2620	{
2621	MemOp ot = decode->op[1].ot;
2622
2623	switch (ot) {
2624	case MO_8:
2625	/* s->T0 already zero-extended */
2626	tcg_gen_ext8u_tl(s->T1, s->T1);
2627	tcg_gen_mul_tl(s->T0, s->T0, s->T1);
2628	gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
2629	tcg_gen_andi_tl(s->T1, s->T0, 0xff00);
2630	decode->cc_dst = s->T0;
2631	decode->cc_src = s->T1;
2632	break;
2633
2634	case MO_16:
2635	/* s->T0 already zero-extended */
2636	tcg_gen_ext16u_tl(s->T1, s->T1);
2637	tcg_gen_mul_tl(s->T0, s->T0, s->T1);
2638	gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
2639	tcg_gen_shri_tl(s->T1, s->T0, 16);
2640	gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1);
2641	decode->cc_dst = s->T0;
2642	decode->cc_src = s->T1;
2643	break;
2644
2645	case MO_32:
2646	#ifdef TARGET_X86_64
2647	/* s->T0 already zero-extended */
2648	tcg_gen_ext32u_tl(s->T1, s->T1);
2649	tcg_gen_mul_tl(s->T0, s->T0, s->T1);
2650	tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0);
2651	tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32);
2652	decode->cc_dst = cpu_regs[R_EAX];
2653	decode->cc_src = cpu_regs[R_EDX];
2654	break;
2655
2656	case MO_64:
2657	#endif
2658	tcg_gen_mulu2_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], s->T0, s->T1);
2659	decode->cc_dst = cpu_regs[R_EAX];
2660	decode->cc_src = cpu_regs[R_EDX];
2661	break;
2662
2663	default:
2664	g_assert_not_reached();
2665	}
2666
2667	decode->cc_op = CC_OP_MULB + ot;
2668	}
2669
2670	static void gen_MULX(DisasContext s, X86DecodedInsn decode)
2671	{
2672	MemOp ot = decode->op[0].ot;
2673
2674	/* low part of result in VEX.vvvv, high in MODRM */
2675	switch (ot) {
2676	case MO_32:
2677	#ifdef TARGET_X86_64
2678	{
2679	TCGv_i32 t0 = tcg_temp_new_i32();
2680	TCGv_i32 t1 = tcg_temp_new_i32();
2681
2682	tcg_gen_trunc_tl_i32(t0, s->T0);
2683	tcg_gen_trunc_tl_i32(t1, s->T1);
2684	tcg_gen_mulu2_i32(t0, t1, t0, t1);
2685	tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], t0);
2686	tcg_gen_extu_i32_tl(s->T0, t1);
2687	break;
2688	}
2689
2690	case MO_64:
2691	#endif
2692	tcg_gen_mulu2_tl(cpu_regs[s->vex_v], s->T0, s->T0, s->T1);
2693	break;
2694
2695	default:
2696	g_assert_not_reached();
2697	}
2698	}
2699
2700	static void gen_NEG(DisasContext s, X86DecodedInsn decode)
2701	{
2702	MemOp ot = decode->op[0].ot;
2703	TCGv oldv = tcg_temp_new();
2704
2705	if (s->prefix & PREFIX_LOCK) {
2706	TCGv newv = tcg_temp_new();
2707	TCGv cmpv = tcg_temp_new();
2708	TCGLabel *label1 = gen_new_label();
2709
2710	gen_set_label(label1);
2711	gen_op_ld_v(s, ot, oldv, s->A0);
2712	tcg_gen_neg_tl(newv, oldv);
2713	tcg_gen_atomic_cmpxchg_tl(cmpv, s->A0, oldv, newv,
2714	s->mem_index, ot \| MO_LE);
2715	tcg_gen_brcond_tl(TCG_COND_NE, oldv, cmpv, label1);
2716	} else {
2717	tcg_gen_mov_tl(oldv, s->T0);
2718	}
2719	tcg_gen_neg_tl(s->T0, oldv);
2720
2721	decode->cc_dst = s->T0;
2722	decode->cc_src = oldv;
2723	s->cc_srcT = tcg_constant_tl(0);
2724	decode->cc_op = CC_OP_SUBB + ot;
2725	}
2726
2727	static void gen_NOT(DisasContext s, X86DecodedInsn decode)
2728	{
2729	MemOp ot = decode->op[0].ot;
2730
2731	if (s->prefix & PREFIX_LOCK) {
2732	tcg_gen_movi_tl(s->T0, ~0);
2733	tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T0,
2734	s->mem_index, ot \| MO_LE);
2735	} else {
2736	tcg_gen_not_tl(s->T0, s->T0);
2737	}
2738	}
2739
2740	static void gen_OR(DisasContext s, X86DecodedInsn decode)
2741	{
2742	MemOp ot = decode->op[1].ot;
2743
2744	if (s->prefix & PREFIX_LOCK) {
2745	tcg_gen_atomic_or_fetch_tl(s->T0, s->A0, s->T1,
2746	s->mem_index, ot \| MO_LE);
2747	} else {
2748	tcg_gen_or_tl(s->T0, s->T0, s->T1);
2749	}
2750	prepare_update1_cc(decode, s, CC_OP_LOGICB + ot);
2751	}
2752
2753	static void gen_OUT(DisasContext s, X86DecodedInsn decode)
2754	{
2755	MemOp ot = decode->op[1].ot;
2756	TCGv_i32 port = tcg_temp_new_i32();
2757	TCGv_i32 value = tcg_temp_new_i32();
2758
2759	tcg_gen_trunc_tl_i32(port, s->T1);
2760	tcg_gen_ext16u_i32(port, port);
2761	if (!gen_check_io(s, ot, port, 0)) {
2762	return;
2763	}
2764	tcg_gen_trunc_tl_i32(value, s->T0);
2765	translator_io_start(&s->base);
2766	gen_helper_out_func(ot, port, value);
2767	gen_bpt_io(s, port, ot);
2768	}
2769
2770	static void gen_OUTS(DisasContext s, X86DecodedInsn decode)
2771	{
2772	MemOp ot = decode->op[1].ot;
2773	TCGv_i32 port = tcg_temp_new_i32();
2774
2775	tcg_gen_trunc_tl_i32(port, s->T1);
2776	tcg_gen_ext16u_i32(port, port);
2777	if (!gen_check_io(s, ot, port, SVM_IOIO_STR_MASK)) {
2778	return;
2779	}
2780
2781	translator_io_start(&s->base);
2782	gen_repz(s, ot, gen_outs);
2783	}
2784
2785	static void gen_PALIGNR(DisasContext s, X86DecodedInsn decode)
2786	{
2787	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
2788	if (!(s->prefix & PREFIX_DATA)) {
2789	gen_helper_palignr_mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
2790	} else if (!s->vex_l) {
2791	gen_helper_palignr_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
2792	} else {
2793	gen_helper_palignr_ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
2794	}
2795	}
2796
2797	static void gen_PANDN(DisasContext s, X86DecodedInsn decode)
2798	{
2799	int vec_len = vector_len(s, decode);
2800
2801	/* Careful, operand order is reversed! */
2802	tcg_gen_gvec_andc(MO_64,
2803	decode->op[0].offset, decode->op[2].offset,
2804	decode->op[1].offset, vec_len, vec_len);
2805	}
2806
2807	static void gen_PAUSE(DisasContext s, X86DecodedInsn decode)
2808	{
2809	gen_update_cc_op(s);
2810	gen_update_eip_next(s);
2811	gen_helper_pause(tcg_env);
2812	s->base.is_jmp = DISAS_NORETURN;
2813	}
2814
2815	static void gen_PCMPESTRI(DisasContext s, X86DecodedInsn decode)
2816	{
2817	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
2818	gen_helper_pcmpestri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm);
2819	assume_cc_op(s, CC_OP_EFLAGS);
2820	}
2821
2822	static void gen_PCMPESTRM(DisasContext s, X86DecodedInsn decode)
2823	{
2824	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
2825	gen_helper_pcmpestrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm);
2826	assume_cc_op(s, CC_OP_EFLAGS);
2827	if ((s->prefix & PREFIX_VEX) && !s->vex_l) {
2828	tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)),
2829	16, 16, 0);
2830	}
2831	}
2832
2833	static void gen_PCMPISTRI(DisasContext s, X86DecodedInsn decode)
2834	{
2835	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
2836	gen_helper_pcmpistri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm);
2837	assume_cc_op(s, CC_OP_EFLAGS);
2838	}
2839
2840	static void gen_PCMPISTRM(DisasContext s, X86DecodedInsn decode)
2841	{
2842	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
2843	gen_helper_pcmpistrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm);
2844	assume_cc_op(s, CC_OP_EFLAGS);
2845	if ((s->prefix & PREFIX_VEX) && !s->vex_l) {
2846	tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)),
2847	16, 16, 0);
2848	}
2849	}
2850
2851	static void gen_PDEP(DisasContext s, X86DecodedInsn decode)
2852	{
2853	gen_helper_pdep(s->T0, s->T0, s->T1);
2854	}
2855
2856	static void gen_PEXT(DisasContext s, X86DecodedInsn decode)
2857	{
2858	gen_helper_pext(s->T0, s->T0, s->T1);
2859	}
2860
2861	static inline void gen_pextr(DisasContext s, X86DecodedInsn decode, MemOp ot)
2862	{
2863	int vec_len = vector_len(s, decode);
2864	int mask = (vec_len >> ot) - 1;
2865	int val = decode->immediate & mask;
2866
2867	switch (ot) {
2868	case MO_8:
2869	tcg_gen_ld8u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val));
2870	break;
2871	case MO_16:
2872	tcg_gen_ld16u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val));
2873	break;
2874	case MO_32:
2875	#ifdef TARGET_X86_64
2876	tcg_gen_ld32u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val));
2877	break;
2878	case MO_64:
2879	#endif
2880	tcg_gen_ld_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val));
2881	break;
2882	default:
2883	g_assert_not_reached();
2884	}
2885	}
2886
2887	static void gen_PEXTRB(DisasContext s, X86DecodedInsn decode)
2888	{
2889	gen_pextr(s, decode, MO_8);
2890	}
2891
2892	static void gen_PEXTRW(DisasContext s, X86DecodedInsn decode)
2893	{
2894	gen_pextr(s, decode, MO_16);
2895	}
2896
2897	static void gen_PEXTR(DisasContext s, X86DecodedInsn decode)
2898	{
2899	MemOp ot = decode->op[0].ot;
2900	gen_pextr(s, decode, ot);
2901	}
2902
2903	static inline void gen_pinsr(DisasContext s, X86DecodedInsn decode, MemOp ot)
2904	{
2905	int vec_len = vector_len(s, decode);
2906	int mask = (vec_len >> ot) - 1;
2907	int val = decode->immediate & mask;
2908
2909	if (decode->op[1].offset != decode->op[0].offset) {
2910	assert(vec_len == 16);
2911	gen_store_sse(s, decode, decode->op[1].offset);
2912	}
2913
2914	switch (ot) {
2915	case MO_8:
2916	tcg_gen_st8_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val));
2917	break;
2918	case MO_16:
2919	tcg_gen_st16_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val));
2920	break;
2921	case MO_32:
2922	#ifdef TARGET_X86_64
2923	tcg_gen_st32_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val));
2924	break;
2925	case MO_64:
2926	#endif
2927	tcg_gen_st_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val));
2928	break;
2929	default:
2930	g_assert_not_reached();
2931	}
2932	}
2933
2934	static void gen_PINSRB(DisasContext s, X86DecodedInsn decode)
2935	{
2936	gen_pinsr(s, decode, MO_8);
2937	}
2938
2939	static void gen_PINSRW(DisasContext s, X86DecodedInsn decode)
2940	{
2941	gen_pinsr(s, decode, MO_16);
2942	}
2943
2944	static void gen_PINSR(DisasContext s, X86DecodedInsn decode)
2945	{
2946	gen_pinsr(s, decode, decode->op[2].ot);
2947	}
2948
2949	static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s)
2950	{
2951	TCGv_i64 t = tcg_temp_new_i64();
2952
2953	tcg_gen_andi_i64(d, s, 0x8080808080808080ull);
2954
2955	/*
2956	* After each shift+or pair:
2957	* 0: a.......b.......c.......d.......e.......f.......g.......h.......
2958	* 7: ab......bc......cd......de......ef......fg......gh......h.......
2959	* 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h.......
2960	* 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h.......
2961	* The result is left in the high bits of the word.
2962	*/
2963	tcg_gen_shli_i64(t, d, 7);
2964	tcg_gen_or_i64(d, d, t);
2965	tcg_gen_shli_i64(t, d, 14);
2966	tcg_gen_or_i64(d, d, t);
2967	tcg_gen_shli_i64(t, d, 28);
2968	tcg_gen_or_i64(d, d, t);
2969	}
2970
2971	static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s)
2972	{
2973	TCGv_vec t = tcg_temp_new_vec_matching(d);
2974	TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80);
2975
2976	/* See above */
2977	tcg_gen_and_vec(vece, d, s, m);
2978	tcg_gen_shli_vec(vece, t, d, 7);
2979	tcg_gen_or_vec(vece, d, d, t);
2980	tcg_gen_shli_vec(vece, t, d, 14);
2981	tcg_gen_or_vec(vece, d, d, t);
2982	tcg_gen_shli_vec(vece, t, d, 28);
2983	tcg_gen_or_vec(vece, d, d, t);
2984	}
2985
2986	static void gen_PMOVMSKB(DisasContext s, X86DecodedInsn decode)
2987	{
2988	static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
2989	static const GVecGen2 g = {
2990	.fni8 = gen_pmovmskb_i64,
2991	.fniv = gen_pmovmskb_vec,
2992	.opt_opc = vecop_list,
2993	.vece = MO_64,
2994	.prefer_i64 = true
2995	};
2996	MemOp ot = decode->op[2].ot;
2997	int vec_len = vector_len(s, decode);
2998	TCGv t = tcg_temp_new();
2999
3000	tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset,
3001	vec_len, vec_len, &g);
3002	tcg_gen_ld8u_tl(s->T0, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1)));
3003	while (vec_len > 8) {
3004	vec_len -= 8;
3005	if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_TL, 0)) {
3006	/*
3007	* Load the next byte of the result into the high byte of T.
3008	* TCG does a similar expansion of deposit to shl+extract2; by
3009	* loading the whole word, the shift left is avoided.
3010	*/
3011	#ifdef TARGET_X86_64
3012	tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8)));
3013	#else
3014	tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4)));
3015	#endif
3016
3017	tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8);
3018	} else {
3019	/*
3020	* The _previous_ value is deposited into bits 8 and higher of t. Because
3021	* those bits are known to be zero after ld8u, this becomes a shift+or
3022	* if deposit is not available.
3023	*/
3024	tcg_gen_ld8u_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1)));
3025	tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8);
3026	}
3027	}
3028	}
3029
3030	static void gen_POP(DisasContext s, X86DecodedInsn decode)
3031	{
3032	X86DecodedOp *op = &decode->op[0];
3033	MemOp ot = gen_pop_T0(s);
3034
3035	assert(ot >= op->ot);
3036	if (op->has_ea \|\| op->unit == X86_OP_SEG) {
3037	/* NOTE: order is important for MMU exceptions */
3038	gen_writeback(s, decode, 0, s->T0);
3039	}
3040
3041	/* NOTE: writing back registers after update is important for pop %sp */
3042	gen_pop_update(s, ot);
3043	}
3044
3045	static void gen_POPA(DisasContext s, X86DecodedInsn decode)
3046	{
3047	gen_popa(s);
3048	}
3049
3050	static void gen_POPCNT(DisasContext s, X86DecodedInsn decode)
3051	{
3052	decode->cc_dst = tcg_temp_new();
3053	decode->cc_op = CC_OP_POPCNT;
3054
3055	tcg_gen_mov_tl(decode->cc_dst, s->T0);
3056	tcg_gen_ctpop_tl(s->T0, s->T0);
3057	}
3058
3059	static void gen_POPF(DisasContext s, X86DecodedInsn decode)
3060	{
3061	MemOp ot;
3062	int mask = TF_MASK \| AC_MASK \| ID_MASK \| NT_MASK;
3063
3064	if (CPL(s) == 0) {
3065	mask \|= IF_MASK \| IOPL_MASK;
3066	} else if (CPL(s) <= IOPL(s)) {
3067	mask \|= IF_MASK;
3068	}
3069	if (s->dflag == MO_16) {
3070	mask &= 0xffff;
3071	}
3072
3073	ot = gen_pop_T0(s);
3074	gen_helper_write_eflags(tcg_env, s->T0, tcg_constant_i32(mask));
3075	gen_pop_update(s, ot);
3076	set_cc_op(s, CC_OP_EFLAGS);
3077	/* abort translation because TF/AC flag may change */
3078	s->base.is_jmp = DISAS_EOB_NEXT;
3079	}
3080
3081	static void gen_PSHUFW(DisasContext s, X86DecodedInsn decode)
3082	{
3083	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
3084	gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm);
3085	}
3086
3087	static void gen_PSRLW_i(DisasContext s, X86DecodedInsn decode)
3088	{
3089	int vec_len = vector_len(s, decode);
3090
3091	if (decode->immediate >= 16) {
3092	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
3093	} else {
3094	tcg_gen_gvec_shri(MO_16,
3095	decode->op[0].offset, decode->op[1].offset,
3096	decode->immediate, vec_len, vec_len);
3097	}
3098	}
3099
3100	static void gen_PSLLW_i(DisasContext s, X86DecodedInsn decode)
3101	{
3102	int vec_len = vector_len(s, decode);
3103
3104	if (decode->immediate >= 16) {
3105	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
3106	} else {
3107	tcg_gen_gvec_shli(MO_16,
3108	decode->op[0].offset, decode->op[1].offset,
3109	decode->immediate, vec_len, vec_len);
3110	}
3111	}
3112
3113	static void gen_PSRAW_i(DisasContext s, X86DecodedInsn decode)
3114	{
3115	int vec_len = vector_len(s, decode);
3116
3117	if (decode->immediate >= 16) {
3118	decode->immediate = 15;
3119	}
3120	tcg_gen_gvec_sari(MO_16,
3121	decode->op[0].offset, decode->op[1].offset,
3122	decode->immediate, vec_len, vec_len);
3123	}
3124
3125	static void gen_PSRLD_i(DisasContext s, X86DecodedInsn decode)
3126	{
3127	int vec_len = vector_len(s, decode);
3128
3129	if (decode->immediate >= 32) {
3130	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
3131	} else {
3132	tcg_gen_gvec_shri(MO_32,
3133	decode->op[0].offset, decode->op[1].offset,
3134	decode->immediate, vec_len, vec_len);
3135	}
3136	}
3137
3138	static void gen_PSLLD_i(DisasContext s, X86DecodedInsn decode)
3139	{
3140	int vec_len = vector_len(s, decode);
3141
3142	if (decode->immediate >= 32) {
3143	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
3144	} else {
3145	tcg_gen_gvec_shli(MO_32,
3146	decode->op[0].offset, decode->op[1].offset,
3147	decode->immediate, vec_len, vec_len);
3148	}
3149	}
3150
3151	static void gen_PSRAD_i(DisasContext s, X86DecodedInsn decode)
3152	{
3153	int vec_len = vector_len(s, decode);
3154
3155	if (decode->immediate >= 32) {
3156	decode->immediate = 31;
3157	}
3158	tcg_gen_gvec_sari(MO_32,
3159	decode->op[0].offset, decode->op[1].offset,
3160	decode->immediate, vec_len, vec_len);
3161	}
3162
3163	static void gen_PSRLQ_i(DisasContext s, X86DecodedInsn decode)
3164	{
3165	int vec_len = vector_len(s, decode);
3166
3167	if (decode->immediate >= 64) {
3168	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
3169	} else {
3170	tcg_gen_gvec_shri(MO_64,
3171	decode->op[0].offset, decode->op[1].offset,
3172	decode->immediate, vec_len, vec_len);
3173	}
3174	}
3175
3176	static void gen_PSLLQ_i(DisasContext s, X86DecodedInsn decode)
3177	{
3178	int vec_len = vector_len(s, decode);
3179
3180	if (decode->immediate >= 64) {
3181	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
3182	} else {
3183	tcg_gen_gvec_shli(MO_64,
3184	decode->op[0].offset, decode->op[1].offset,
3185	decode->immediate, vec_len, vec_len);
3186	}
3187	}
3188
3189	static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len)
3190	{
3191	MemOp ot = vec_len == 16 ? MO_128 : MO_256;
3192	TCGv_i32 imm_v = tcg_constant8u_i32(imm);
3193	TCGv_ptr ptr = tcg_temp_new_ptr();
3194
3195	tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot),
3196	vec_len, vec_len, 0);
3197
3198	tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_t0));
3199	tcg_gen_st_i32(imm_v, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0)));
3200	return ptr;
3201	}
3202
3203	static void gen_PSRLDQ_i(DisasContext s, X86DecodedInsn decode)
3204	{
3205	int vec_len = vector_len(s, decode);
3206	TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len);
3207
3208	if (s->vex_l) {
3209	gen_helper_psrldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec);
3210	} else {
3211	gen_helper_psrldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec);
3212	}
3213	}
3214
3215	static void gen_PSLLDQ_i(DisasContext s, X86DecodedInsn decode)
3216	{
3217	int vec_len = vector_len(s, decode);
3218	TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len);
3219
3220	if (s->vex_l) {
3221	gen_helper_pslldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec);
3222	} else {
3223	gen_helper_pslldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec);
3224	}
3225	}
3226
3227	static void gen_PUSH(DisasContext s, X86DecodedInsn decode)
3228	{
3229	gen_push_v(s, s->T0);
3230	}
3231
3232	static void gen_PUSHA(DisasContext s, X86DecodedInsn decode)
3233	{
3234	gen_pusha(s);
3235	}
3236
3237	static void gen_PUSHF(DisasContext s, X86DecodedInsn decode)
3238	{
3239	gen_update_cc_op(s);
3240	gen_helper_read_eflags(s->T0, tcg_env);
3241	gen_push_v(s, s->T0);
3242	assume_cc_op(s, CC_OP_EFLAGS);
3243	}
3244
3245	static MemOp gen_shift_count_1(DisasContext s, X86DecodedInsn decode,
3246	bool can_be_zero, TCGv count, int unit,
3247	int mod)
3248	{
3249	MemOp ot = decode->op[0].ot;
3250	int mask = (ot <= MO_32 ? 0x1f : 0x3f);
3251
3252	*can_be_zero = false;
3253	switch (unit) {
3254	case X86_OP_INT:
3255	*count = tcg_temp_new();
3256	tcg_gen_andi_tl(*count, cpu_regs[R_ECX], mask);
3257
3258	if (mod < mask) {
3259	TCGv temp = tcg_temp_new();
3260	assert(mod * 4 >= mask);
3261	if (mod * 2 < mask) {
3262	tcg_gen_subi_tl(temp, count, mod 2);
3263	tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count);
3264	}
3265	tcg_gen_subi_tl(temp, *count, mod);
3266	tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count);
3267	}
3268	*can_be_zero = true;
3269	break;
3270
3271	case X86_OP_IMM:
3272	decode->immediate &= mask;
3273	if (mod < mask) {
3274	decode->immediate %= mod;
3275	}
3276	if (decode->immediate == 0) {
3277	*count = NULL;
3278	break;
3279	}
3280	*count = tcg_temp_new();
3281	tcg_gen_movi_tl(*count, decode->immediate);
3282	break;
3283
3284	case X86_OP_SKIP:
3285	*count = tcg_temp_new();
3286	tcg_gen_movi_tl(*count, 1);
3287	break;
3288
3289	default:
3290	g_assert_not_reached();
3291	}
3292
3293	return ot;
3294	}
3295
3296	static MemOp gen_shift_count(DisasContext s, X86DecodedInsn decode,
3297	bool can_be_zero, TCGv count, int unit)
3298	{
3299	return gen_shift_count_1(s, decode, can_be_zero, count, unit,
3300	INT_MAX);
3301	}
3302
3303	/*
3304	* Compute existing flags in decode->cc_src, for gen_* functions that wants
3305	* to set the cc_op set to CC_OP_ADCOX. In particular, this allows rotate
3306	* operations to compute the carry in decode->cc_dst and the overflow in
3307	* decode->cc_src2.
3308	*
3309	* If need_flags is true, decode->cc_dst and decode->cc_src2 are preloaded
3310	* with the value of CF and OF before the instruction, so that it is possible
3311	* to keep the flags unmodified.
3312	*
3313	* Return true if carry could be made available cheaply as a 1-bit value in
3314	* decode->cc_dst (trying a bit harder if want_carry is true). If false is
3315	* returned, decode->cc_dst is uninitialized and the carry is only available
3316	* as bit 0 of decode->cc_src.
3317	*/
3318	static bool gen_eflags_adcox(DisasContext s, X86DecodedInsn decode, bool want_carry, bool need_flags)
3319	{
3320	bool got_cf = false;
3321	bool got_of = false;
3322
3323	decode->cc_dst = tcg_temp_new();
3324	decode->cc_src = tcg_temp_new();
3325	decode->cc_src2 = tcg_temp_new();
3326	decode->cc_op = CC_OP_ADCOX;
3327
3328	/* A lot more cc_ops could be "optimized" to avoid the extracts at
3329	* the end (INC/DEC, BMILG, MUL), but they are all really unlikely
3330	* to be followed by rotations within the same basic block.
3331	*/
3332	switch (s->cc_op) {
3333	case CC_OP_ADCX:
3334	case CC_OP_ADOX:
3335	case CC_OP_ADCOX:
3336	/* No need to compute the full EFLAGS, CF/OF are already isolated. */
3337	if (s->cc_op != CC_OP_ADCX && need_flags) {
3338	tcg_gen_mov_tl(decode->cc_src2, cpu_cc_src2);
3339	got_of = true;
3340	}
3341	if (s->cc_op != CC_OP_ADOX && (want_carry \|\| need_flags)) {
3342	tcg_gen_mov_tl(decode->cc_dst, cpu_cc_dst);
3343	got_cf = true;
3344	}
3345	/* fallthrough */
3346	case CC_OP_EFLAGS:
3347	tcg_gen_mov_tl(decode->cc_src, cpu_cc_src);
3348	break;
3349
3350	case CC_OP_LOGICB ... CC_OP_LOGICQ:
3351	/* CF and OF are zero, do it just because it's easy. */
3352	gen_mov_eflags(s, decode->cc_src);
3353	if (need_flags) {
3354	tcg_gen_movi_tl(decode->cc_src2, 0);
3355	got_of = true;
3356	}
3357	if (want_carry \|\| need_flags) {
3358	tcg_gen_movi_tl(decode->cc_dst, 0);
3359	got_cf = true;
3360	}
3361	break;
3362
3363	case CC_OP_SARB ... CC_OP_SARQ:
3364	/*
3365	* SHR/RCR/SHR/RCR/... is a relatively common occurrence of RCR.
3366	* By computing CF without using eflags, the calls to cc_compute_all
3367	* can be eliminated as dead code (except for the last RCR).
3368	*/
3369	if (want_carry \|\| need_flags) {
3370	tcg_gen_andi_tl(decode->cc_dst, cpu_cc_src, 1);
3371	got_cf = true;
3372	}
3373	gen_mov_eflags(s, decode->cc_src);
3374	break;
3375
3376	case CC_OP_SHLB ... CC_OP_SHLQ:
3377	/*
3378	* Likewise for SHL/RCL/SHL/RCL/... but, if CF is not in the sign
3379	* bit, we might as well fish CF out of EFLAGS and save a shift.
3380	*/
3381	if (want_carry && (!need_flags \|\| s->cc_op == CC_OP_SHLB + MO_TL)) {
3382	MemOp size = cc_op_size(s->cc_op);
3383	tcg_gen_shri_tl(decode->cc_dst, cpu_cc_src, (8 << size) - 1);
3384	got_cf = true;
3385	}
3386	gen_mov_eflags(s, decode->cc_src);
3387	break;
3388
3389	default:
3390	gen_mov_eflags(s, decode->cc_src);
3391	break;
3392	}
3393
3394	if (need_flags) {
3395	/* If the flags could be left unmodified, always load them. */
3396	if (!got_of) {
3397	tcg_gen_extract_tl(decode->cc_src2, decode->cc_src, ctz32(CC_O), 1);
3398	got_of = true;
3399	}
3400	if (!got_cf) {
3401	tcg_gen_extract_tl(decode->cc_dst, decode->cc_src, ctz32(CC_C), 1);
3402	got_cf = true;
3403	}
3404	}
3405	return got_cf;
3406	}
3407
3408	static void gen_rot_overflow(X86DecodedInsn *decode, TCGv result, TCGv old,
3409	bool can_be_zero, TCGv count)
3410	{
3411	MemOp ot = decode->op[0].ot;
3412	TCGv temp = can_be_zero ? tcg_temp_new() : decode->cc_src2;
3413
3414	tcg_gen_xor_tl(temp, old, result);
3415	tcg_gen_extract_tl(temp, temp, (8 << ot) - 1, 1);
3416	if (can_be_zero) {
3417	tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src2, count, tcg_constant_tl(0),
3418	decode->cc_src2, temp);
3419	}
3420	}
3421
3422	/*
3423	* RCx operations are invariant modulo 8*operand_size+1. For 8 and 16-bit operands,
3424	* this is less than 0x1f (the mask applied by gen_shift_count) so reduce further.
3425	* A count that is a nonzero multiple of (8 << op) + 1 reduces to 0 and leaves
3426	* CF and OF unmodified (confirmed against hardware that rcl $9,%al behaves
3427	* exactly like rcl $0,%al).
3428	*/
3429	static MemOp gen_rotc_count(DisasContext s, X86DecodedInsn decode,
3430	bool can_be_zero, TCGv count, int unit)
3431	{
3432	MemOp ot = decode->op[0].ot;
3433	return gen_shift_count_1(s, decode, can_be_zero, count, unit,
3434	(8 << ot) + 1);
3435	}
3436
3437	/*
3438	* The idea here is that the bit to the right of the new bit 0 is the
3439	* new carry, and the bit to the right of the old bit 0 is the old carry.
3440	* Just like a regular rotation, the result of the rotation is composed
3441	* from a right shifted part and a left shifted part of s->T0. The new carry
3442	* is extracted from the right-shifted portion, and the old carry is
3443	* inserted at the end of the left-shifted portion.
3444	*
3445	* Because of the separate shifts involving the carry, gen_RCL and gen_RCR
3446	* mostly operate on count-1. This also comes in handy when computing
3447	* length - count, because (length-1) - (count-1) can be computed with
3448	* a XOR, and that is commutative unlike subtraction.
3449	*/
3450	static void gen_RCL(DisasContext s, X86DecodedInsn decode)
3451	{
3452	bool have_1bit_cin, can_be_zero;
3453	TCGv count;
3454	TCGLabel *zero_label = NULL;
3455	MemOp ot = gen_rotc_count(s, decode, &can_be_zero, &count, decode->op[2].unit);
3456	TCGv low, high, low_count;
3457
3458	if (!count) {
3459	return;
3460	}
3461
3462	low = tcg_temp_new();
3463	high = tcg_temp_new();
3464	low_count = tcg_temp_new();
3465
3466	have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero);
3467	if (can_be_zero) {
3468	zero_label = gen_new_label();
3469	tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label);
3470	}
3471
3472	/* Compute high part, including incoming carry. */
3473	if (!have_1bit_cin \|\| tcg_op_deposit_valid(TCG_TYPE_TL, 1, TARGET_LONG_BITS - 1)) {
3474	/* high = (T0 << 1) \| cin */
3475	TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src;
3476	tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1);
3477	} else {
3478	/* Same as above but without deposit; cin in cc_dst. */
3479	tcg_gen_add_tl(high, s->T0, decode->cc_dst);
3480	tcg_gen_add_tl(high, high, s->T0);
3481	}
3482	tcg_gen_subi_tl(count, count, 1);
3483	tcg_gen_shl_tl(high, high, count);
3484
3485	/* Compute low part and outgoing carry, incoming s->T0 is zero extended */
3486	tcg_gen_xori_tl(low_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */
3487	tcg_gen_shr_tl(low, s->T0, low_count);
3488	tcg_gen_andi_tl(decode->cc_dst, low, 1);
3489	tcg_gen_shri_tl(low, low, 1);
3490
3491	/* Compute result and outgoing overflow */
3492	tcg_gen_mov_tl(decode->cc_src2, s->T0);
3493	tcg_gen_or_tl(s->T0, low, high);
3494	gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL);
3495
3496	if (zero_label) {
3497	gen_set_label(zero_label);
3498	}
3499	}
3500
3501	static void gen_RCR(DisasContext s, X86DecodedInsn decode)
3502	{
3503	bool have_1bit_cin, can_be_zero;
3504	TCGv count;
3505	TCGLabel *zero_label = NULL;
3506	MemOp ot = gen_rotc_count(s, decode, &can_be_zero, &count, decode->op[2].unit);
3507	TCGv low, high, high_count;
3508
3509	if (!count) {
3510	return;
3511	}
3512
3513	low = tcg_temp_new();
3514	high = tcg_temp_new();
3515	high_count = tcg_temp_new();
3516
3517	have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero);
3518	if (can_be_zero) {
3519	zero_label = gen_new_label();
3520	tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label);
3521	}
3522
3523	/* Save incoming carry into high, it will be shifted later. */
3524	if (!have_1bit_cin \|\| tcg_op_deposit_valid(TCG_TYPE_TL, 1, TARGET_LONG_BITS - 1)) {
3525	TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src;
3526	tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1);
3527	} else {
3528	/* Same as above but without deposit; cin in cc_dst. */
3529	tcg_gen_add_tl(high, s->T0, decode->cc_dst);
3530	tcg_gen_add_tl(high, high, s->T0);
3531	}
3532
3533	/* Compute low part and outgoing carry, incoming s->T0 is zero extended */
3534	tcg_gen_subi_tl(count, count, 1);
3535	tcg_gen_shr_tl(low, s->T0, count);
3536	tcg_gen_andi_tl(decode->cc_dst, low, 1);
3537	tcg_gen_shri_tl(low, low, 1);
3538
3539	/* Move high part to the right position */
3540	tcg_gen_xori_tl(high_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */
3541	tcg_gen_shl_tl(high, high, high_count);
3542
3543	/* Compute result and outgoing overflow */
3544	tcg_gen_mov_tl(decode->cc_src2, s->T0);
3545	tcg_gen_or_tl(s->T0, low, high);
3546	gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL);
3547
3548	if (zero_label) {
3549	gen_set_label(zero_label);
3550	}
3551	}
3552
3553	#ifdef CONFIG_USER_ONLY
3554	static void gen_unreachable(DisasContext s, X86DecodedInsn decode)
3555	{
3556	g_assert_not_reached();
3557	}
3558	#endif
3559
3560	#ifndef CONFIG_USER_ONLY
3561	static void gen_RDMSR(DisasContext s, X86DecodedInsn decode)
3562	{
3563	gen_update_cc_op(s);
3564	gen_update_eip_cur(s);
3565	gen_helper_rdmsr(tcg_env);
3566	}
3567	#else
3568	#define gen_RDMSR gen_unreachable
3569	#endif
3570
3571	static void gen_RDPMC(DisasContext s, X86DecodedInsn decode)
3572	{
3573	gen_update_cc_op(s);
3574	gen_update_eip_cur(s);
3575	translator_io_start(&s->base);
3576	gen_helper_rdpmc(tcg_env);
3577	s->base.is_jmp = DISAS_NORETURN;
3578	}
3579
3580	static void gen_RDTSC(DisasContext s, X86DecodedInsn decode)
3581	{
3582	gen_update_cc_op(s);
3583	gen_update_eip_cur(s);
3584	translator_io_start(&s->base);
3585	gen_helper_rdtsc(tcg_env);
3586	}
3587
3588	static void gen_RDxxBASE(DisasContext s, X86DecodedInsn decode)
3589	{
3590	TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS];
3591
3592	/* Preserve hflags bits by testing CR4 at runtime. */
3593	gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK));
3594	tcg_gen_mov_tl(s->T0, base);
3595	}
3596
3597	static void gen_RET(DisasContext s, X86DecodedInsn decode)
3598	{
3599	int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0;
3600
3601	MemOp ot = gen_pop_T0(s);
3602	gen_stack_update(s, adjust + (1 << ot));
3603	gen_op_jmp_v(s, s->T0);
3604	gen_bnd_jmp(s);
3605	s->base.is_jmp = DISAS_JUMP;
3606	}
3607
3608	static void gen_RETF(DisasContext s, X86DecodedInsn decode)
3609	{
3610	int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0;
3611
3612	if (!PE(s) \|\| VM86(s)) {
3613	gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], 0);
3614	/* pop offset */
3615	gen_op_ld_v(s, s->dflag, s->T0, s->A0);
3616	/* NOTE: keeping EIP updated is not a problem in case of
3617	exception */
3618	gen_op_jmp_v(s, s->T0);
3619	/* pop selector */
3620	gen_add_A0_im(s, 1 << s->dflag);
3621	gen_op_ld_v(s, s->dflag, s->T0, s->A0);
3622	gen_op_movl_seg_real(s, R_CS, s->T0);
3623	/* add stack offset */
3624	gen_stack_update(s, adjust + (2 << s->dflag));
3625	} else {
3626	gen_update_cc_op(s);
3627	gen_update_eip_cur(s);
3628	gen_helper_lret_protected(tcg_env, tcg_constant_i32(s->dflag - 1),
3629	tcg_constant_i32(adjust));
3630	}
3631	s->base.is_jmp = DISAS_EOB_ONLY;
3632	}
3633
3634	/*
3635	* Return non-NULL if a 32-bit rotate works, after possibly replicating the input.
3636	* The input has already been zero-extended upon operand decode.
3637	*/
3638	static TCGv_i32 gen_rot_replicate(MemOp ot, TCGv in)
3639	{
3640	TCGv_i32 temp;
3641	switch (ot) {
3642	case MO_8:
3643	temp = tcg_temp_new_i32();
3644	tcg_gen_trunc_tl_i32(temp, in);
3645	tcg_gen_muli_i32(temp, temp, 0x01010101);
3646	return temp;
3647
3648	case MO_16:
3649	temp = tcg_temp_new_i32();
3650	tcg_gen_trunc_tl_i32(temp, in);
3651	tcg_gen_deposit_i32(temp, temp, temp, 16, 16);
3652	return temp;
3653
3654	#ifdef TARGET_X86_64
3655	case MO_32:
3656	temp = tcg_temp_new_i32();
3657	tcg_gen_trunc_tl_i32(temp, in);
3658	return temp;
3659	#endif
3660
3661	default:
3662	return NULL;
3663	}
3664	}
3665
3666	static void gen_rot_carry(X86DecodedInsn *decode, TCGv result,
3667	bool can_be_zero, TCGv count, int bit)
3668	{
3669	if (!can_be_zero) {
3670	tcg_gen_extract_tl(decode->cc_dst, result, bit, 1);
3671	} else {
3672	TCGv temp = tcg_temp_new();
3673	tcg_gen_extract_tl(temp, result, bit, 1);
3674	tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0),
3675	decode->cc_dst, temp);
3676	}
3677	}
3678
3679	static void gen_ROL(DisasContext s, X86DecodedInsn decode)
3680	{
3681	bool can_be_zero;
3682	TCGv count;
3683	MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit);
3684	TCGv_i32 temp32, count32;
3685	TCGv old = tcg_temp_new();
3686
3687	if (!count) {
3688	return;
3689	}
3690
3691	gen_eflags_adcox(s, decode, false, can_be_zero);
3692	tcg_gen_mov_tl(old, s->T0);
3693	temp32 = gen_rot_replicate(ot, s->T0);
3694	if (temp32) {
3695	count32 = tcg_temp_new_i32();
3696	tcg_gen_trunc_tl_i32(count32, count);
3697	tcg_gen_rotl_i32(temp32, temp32, count32);
3698	/* Zero extend to facilitate later optimization. */
3699	tcg_gen_extu_i32_tl(s->T0, temp32);
3700	} else {
3701	tcg_gen_rotl_tl(s->T0, s->T0, count);
3702	}
3703	gen_rot_carry(decode, s->T0, can_be_zero, count, 0);
3704	gen_rot_overflow(decode, s->T0, old, can_be_zero, count);
3705	}
3706
3707	static void gen_ROR(DisasContext s, X86DecodedInsn decode)
3708	{
3709	bool can_be_zero;
3710	TCGv count;
3711	MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit);
3712	TCGv_i32 temp32, count32;
3713	TCGv old = tcg_temp_new();
3714
3715	if (!count) {
3716	return;
3717	}
3718
3719	gen_eflags_adcox(s, decode, false, can_be_zero);
3720	tcg_gen_mov_tl(old, s->T0);
3721	temp32 = gen_rot_replicate(ot, s->T0);
3722	if (temp32) {
3723	count32 = tcg_temp_new_i32();
3724	tcg_gen_trunc_tl_i32(count32, count);
3725	tcg_gen_rotr_i32(temp32, temp32, count32);
3726	/* Zero extend to facilitate later optimization. */
3727	tcg_gen_extu_i32_tl(s->T0, temp32);
3728	gen_rot_carry(decode, s->T0, can_be_zero, count, 31);
3729	} else {
3730	tcg_gen_rotr_tl(s->T0, s->T0, count);
3731	gen_rot_carry(decode, s->T0, can_be_zero, count, TARGET_LONG_BITS - 1);
3732	}
3733	gen_rot_overflow(decode, s->T0, old, can_be_zero, count);
3734	}
3735
3736	static void gen_RORX(DisasContext s, X86DecodedInsn decode)
3737	{
3738	MemOp ot = decode->op[0].ot;
3739	int mask = ot == MO_64 ? 63 : 31;
3740	int b = decode->immediate & mask;
3741
3742	switch (ot) {
3743	case MO_32:
3744	#ifdef TARGET_X86_64
3745	{
3746	TCGv_i32 tmp = tcg_temp_new_i32();
3747
3748	tcg_gen_trunc_tl_i32(tmp, s->T0);
3749	tcg_gen_rotri_i32(tmp, tmp, b);
3750	tcg_gen_extu_i32_tl(s->T0, tmp);
3751	break;
3752	}
3753
3754	case MO_64:
3755	#endif
3756	tcg_gen_rotri_tl(s->T0, s->T0, b);
3757	break;
3758
3759	default:
3760	g_assert_not_reached();
3761	}
3762	}
3763
3764	#ifndef CONFIG_USER_ONLY
3765	static void gen_RSM(DisasContext s, X86DecodedInsn decode)
3766	{
3767	gen_helper_rsm(tcg_env);
3768	assume_cc_op(s, CC_OP_EFLAGS);
3769	s->base.is_jmp = DISAS_EOB_ONLY;
3770	}
3771	#else
3772	#define gen_RSM gen_UD
3773	#endif
3774
3775	static void gen_SAHF(DisasContext s, X86DecodedInsn decode)
3776	{
3777	if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) {
3778	return gen_illegal_opcode(s);
3779	}
3780	tcg_gen_shri_tl(s->T0, cpu_regs[R_EAX], 8);
3781	gen_neg_setcc(s, JCC_O << 1, s->T1);
3782	tcg_gen_andi_tl(s->T1, s->T1, CC_O);
3783	tcg_gen_andi_tl(s->T0, s->T0, CC_S \| CC_Z \| CC_A \| CC_P \| CC_C);
3784	tcg_gen_or_tl(s->T0, s->T0, s->T1);
3785
3786	decode->cc_src = s->T0;
3787	decode->cc_op = CC_OP_EFLAGS;
3788	}
3789
3790	static void gen_SALC(DisasContext s, X86DecodedInsn decode)
3791	{
3792	gen_compute_eflags_c(s, s->T0);
3793	tcg_gen_neg_tl(s->T0, s->T0);
3794	}
3795
3796	static void gen_shift_dynamic_flags(DisasContext s, X86DecodedInsn decode, TCGv count, CCOp cc_op)
3797	{
3798	TCGv_i32 count32 = tcg_temp_new_i32();
3799	TCGv_i32 old_cc_op;
3800
3801	decode->cc_op = CC_OP_DYNAMIC;
3802	decode->cc_op_dynamic = tcg_temp_new_i32();
3803
3804	assert(decode->cc_dst == s->T0);
3805	if (cc_op_live(s->cc_op) & USES_CC_DST) {
3806	decode->cc_dst = tcg_temp_new();
3807	tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0),
3808	cpu_cc_dst, s->T0);
3809	}
3810
3811	if (cc_op_live(s->cc_op) & USES_CC_SRC) {
3812	tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src, count, tcg_constant_tl(0),
3813	cpu_cc_src, decode->cc_src);
3814	}
3815
3816	tcg_gen_trunc_tl_i32(count32, count);
3817	if (s->cc_op == CC_OP_DYNAMIC) {
3818	old_cc_op = cpu_cc_op;
3819	} else {
3820	old_cc_op = tcg_constant_i32(s->cc_op);
3821	}
3822	tcg_gen_movcond_i32(TCG_COND_EQ, decode->cc_op_dynamic, count32, tcg_constant_i32(0),
3823	old_cc_op, tcg_constant_i32(cc_op));
3824	}
3825
3826	static void gen_SAR(DisasContext s, X86DecodedInsn decode)
3827	{
3828	bool can_be_zero;
3829	TCGv count;
3830	MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit);
3831
3832	if (!count) {
3833	return;
3834	}
3835
3836	decode->cc_dst = s->T0;
3837	decode->cc_src = tcg_temp_new();
3838	tcg_gen_subi_tl(decode->cc_src, count, 1);
3839	tcg_gen_sar_tl(decode->cc_src, s->T0, decode->cc_src);
3840	tcg_gen_sar_tl(s->T0, s->T0, count);
3841	if (can_be_zero) {
3842	gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot);
3843	} else {
3844	decode->cc_op = CC_OP_SARB + ot;
3845	}
3846	}
3847
3848	static void gen_SARX(DisasContext s, X86DecodedInsn decode)
3849	{
3850	MemOp ot = decode->op[0].ot;
3851	int mask;
3852
3853	mask = ot == MO_64 ? 63 : 31;
3854	tcg_gen_andi_tl(s->T1, s->T1, mask);
3855	tcg_gen_sar_tl(s->T0, s->T0, s->T1);
3856	}
3857
3858	static void gen_SUB(DisasContext s, X86DecodedInsn decode);
3859	static void gen_SBB(DisasContext s, X86DecodedInsn decode)
3860	{
3861	MemOp ot = decode->op[0].ot;
3862	TCGv c_in;
3863
3864	/*
3865	* Try to avoid CC_OP_SBB by transforming as follows:
3866	* CC_SBB: src1 = dst + c_in, src2 = 0, src3 = c_in
3867	* CC_SUB: src1 = dst + c_in, src2 = c_in (no src3)
3868	*
3869	* In general src2 vs. src3 matters when computing AF and OF, but not here:
3870	* - AF is bit 4 of dst^src1^src2, which is bit 4 of dst^src1 in both cases
3871	* - OF is a function of the two MSBs, and in both cases they are zero for src2
3872	*/
3873	if (decode->e.op2 == X86_TYPE_I && decode->immediate == 0) {
3874	gen_compute_eflags_c(s, s->T1);
3875	gen_SUB(s, decode);
3876	return;
3877	}
3878
3879	/* SBB x,x has its own CCOp so that's even easier. */
3880	if (decode->e.op2 != X86_TYPE_I && !decode->op[0].has_ea && decode->op[0].n == decode->op[2].n) {
3881	gen_neg_setcc(s, JCC_B << 1, s->T0);
3882	prepare_update1_cc(decode, s, CC_OP_SBB_SELF);
3883	return;
3884	}
3885
3886	c_in = tcg_temp_new();
3887	gen_compute_eflags_c(s, c_in);
3888
3889	if (s->prefix & PREFIX_LOCK) {
3890	tcg_gen_add_tl(s->T0, s->T1, c_in);
3891	tcg_gen_neg_tl(s->T0, s->T0);
3892	tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0,
3893	s->mem_index, ot \| MO_LE);
3894	} else {
3895	tcg_gen_sub_tl(s->T0, s->T0, s->T1);
3896	tcg_gen_sub_tl(s->T0, s->T0, c_in);
3897	}
3898	prepare_update3_cc(decode, s, CC_OP_SBBB + ot, c_in);
3899	}
3900
3901	static void gen_SCAS(DisasContext s, X86DecodedInsn decode)
3902	{
3903	MemOp ot = decode->op[2].ot;
3904	gen_repz_nz(s, ot, gen_scas);
3905	}
3906
3907	static void gen_SETcc(DisasContext s, X86DecodedInsn decode)
3908	{
3909	gen_setcc(s, decode->b & 0xf, s->T0);
3910	}
3911
3912	static void gen_SFENCE(DisasContext s, X86DecodedInsn decode)
3913	{
3914	tcg_gen_mb(TCG_MO_ST_ST \| TCG_BAR_SC);
3915	}
3916
3917	static void gen_SHA1NEXTE(DisasContext s, X86DecodedInsn decode)
3918	{
3919	gen_helper_sha1nexte(OP_PTR0, OP_PTR1, OP_PTR2);
3920	}
3921
3922	static void gen_SHA1MSG1(DisasContext s, X86DecodedInsn decode)
3923	{
3924	gen_helper_sha1msg1(OP_PTR0, OP_PTR1, OP_PTR2);
3925	}
3926
3927	static void gen_SHA1MSG2(DisasContext s, X86DecodedInsn decode)
3928	{
3929	gen_helper_sha1msg2(OP_PTR0, OP_PTR1, OP_PTR2);
3930	}
3931
3932	static void gen_SHA1RNDS4(DisasContext s, X86DecodedInsn decode)
3933	{
3934	switch(decode->immediate & 3) {
3935	case 0:
3936	gen_helper_sha1rnds4_f0(OP_PTR0, OP_PTR0, OP_PTR1);
3937	break;
3938	case 1:
3939	gen_helper_sha1rnds4_f1(OP_PTR0, OP_PTR0, OP_PTR1);
3940	break;
3941	case 2:
3942	gen_helper_sha1rnds4_f2(OP_PTR0, OP_PTR0, OP_PTR1);
3943	break;
3944	case 3:
3945	gen_helper_sha1rnds4_f3(OP_PTR0, OP_PTR0, OP_PTR1);
3946	break;
3947	}
3948	}
3949
3950	static void gen_SHA256MSG1(DisasContext s, X86DecodedInsn decode)
3951	{
3952	gen_helper_sha256msg1(OP_PTR0, OP_PTR1, OP_PTR2);
3953	}
3954
3955	static void gen_SHA256MSG2(DisasContext s, X86DecodedInsn decode)
3956	{
3957	gen_helper_sha256msg2(OP_PTR0, OP_PTR1, OP_PTR2);
3958	}
3959
3960	static void gen_SHA256RNDS2(DisasContext s, X86DecodedInsn decode)
3961	{
3962	TCGv_i32 wk0 = tcg_temp_new_i32();
3963	TCGv_i32 wk1 = tcg_temp_new_i32();
3964
3965	tcg_gen_ld_i32(wk0, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(0)));
3966	tcg_gen_ld_i32(wk1, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(1)));
3967
3968	gen_helper_sha256rnds2(OP_PTR0, OP_PTR1, OP_PTR2, wk0, wk1);
3969	}
3970
3971	static void gen_SHL(DisasContext s, X86DecodedInsn decode)
3972	{
3973	bool can_be_zero;
3974	TCGv count;
3975	MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit);
3976
3977	if (!count) {
3978	return;
3979	}
3980
3981	decode->cc_dst = s->T0;
3982	decode->cc_src = tcg_temp_new();
3983	tcg_gen_subi_tl(decode->cc_src, count, 1);
3984	tcg_gen_shl_tl(decode->cc_src, s->T0, decode->cc_src);
3985	tcg_gen_shl_tl(s->T0, s->T0, count);
3986	if (can_be_zero) {
3987	gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot);
3988	} else {
3989	decode->cc_op = CC_OP_SHLB + ot;
3990	}
3991	}
3992
3993	static void gen_SHLD(DisasContext s, X86DecodedInsn decode)
3994	{
3995	bool can_be_zero;
3996	TCGv count;
3997	int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT;
3998	MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit);
3999
4000	if (!count) {
4001	return;
4002	}
4003
4004	decode->cc_dst = s->T0;
4005	decode->cc_src = gen_shiftd_rm_T1(s, ot, false, count);
4006	if (can_be_zero) {
4007	gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot);
4008	} else {
4009	decode->cc_op = CC_OP_SHLB + ot;
4010	}
4011	}
4012
4013	static void gen_SHLX(DisasContext s, X86DecodedInsn decode)
4014	{
4015	MemOp ot = decode->op[0].ot;
4016	int mask;
4017
4018	mask = ot == MO_64 ? 63 : 31;
4019	tcg_gen_andi_tl(s->T1, s->T1, mask);
4020	tcg_gen_shl_tl(s->T0, s->T0, s->T1);
4021	}
4022
4023	static void gen_SHR(DisasContext s, X86DecodedInsn decode)
4024	{
4025	bool can_be_zero;
4026	TCGv count;
4027	MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit);
4028
4029	if (!count) {
4030	return;
4031	}
4032
4033	decode->cc_dst = s->T0;
4034	decode->cc_src = tcg_temp_new();
4035	tcg_gen_subi_tl(decode->cc_src, count, 1);
4036	tcg_gen_shr_tl(decode->cc_src, s->T0, decode->cc_src);
4037	tcg_gen_shr_tl(s->T0, s->T0, count);
4038	if (can_be_zero) {
4039	gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot);
4040	} else {
4041	decode->cc_op = CC_OP_SARB + ot;
4042	}
4043	}
4044
4045	static void gen_SHRD(DisasContext s, X86DecodedInsn decode)
4046	{
4047	bool can_be_zero;
4048	TCGv count;
4049	int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT;
4050	MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit);
4051
4052	if (!count) {
4053	return;
4054	}
4055
4056	decode->cc_dst = s->T0;
4057	decode->cc_src = gen_shiftd_rm_T1(s, ot, true, count);
4058	if (can_be_zero) {
4059	gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot);
4060	} else {
4061	decode->cc_op = CC_OP_SARB + ot;
4062	}
4063	}
4064
4065	static void gen_SHRX(DisasContext s, X86DecodedInsn decode)
4066	{
4067	MemOp ot = decode->op[0].ot;
4068	int mask;
4069
4070	mask = ot == MO_64 ? 63 : 31;
4071	tcg_gen_andi_tl(s->T1, s->T1, mask);
4072	tcg_gen_shr_tl(s->T0, s->T0, s->T1);
4073	}
4074
4075	static void gen_STC(DisasContext s, X86DecodedInsn decode)
4076	{
4077	gen_compute_eflags(s);
4078	tcg_gen_ori_tl(cpu_cc_src, cpu_cc_src, CC_C);
4079	}
4080
4081	static void gen_STD(DisasContext s, X86DecodedInsn decode)
4082	{
4083	tcg_gen_st_i32(tcg_constant_i32(-1), tcg_env, offsetof(CPUX86State, df));
4084	}
4085
4086	static void gen_STI(DisasContext s, X86DecodedInsn decode)
4087	{
4088	gen_set_eflags(s, IF_MASK);
4089	s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ;
4090	}
4091
4092	static void gen_VAESKEYGEN(DisasContext s, X86DecodedInsn decode)
4093	{
4094	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
4095	assert(!s->vex_l);
4096	gen_helper_aeskeygenassist_xmm(tcg_env, OP_PTR0, OP_PTR1, imm);
4097	}
4098
4099	static void gen_STMXCSR(DisasContext s, X86DecodedInsn decode)
4100	{
4101	gen_helper_update_mxcsr(tcg_env);
4102	tcg_gen_ld32u_tl(s->T0, tcg_env, offsetof(CPUX86State, mxcsr));
4103	}
4104
4105	static void gen_STOS(DisasContext s, X86DecodedInsn decode)
4106	{
4107	MemOp ot = decode->op[1].ot;
4108	gen_repz(s, ot, gen_stos);
4109	}
4110
4111	static void gen_SUB(DisasContext s, X86DecodedInsn decode)
4112	{
4113	MemOp ot = decode->op[1].ot;
4114
4115	s->cc_srcT = tcg_temp_new();
4116	if (s->prefix & PREFIX_LOCK) {
4117	tcg_gen_neg_tl(s->T0, s->T1);
4118	tcg_gen_atomic_fetch_add_tl(s->cc_srcT, s->A0, s->T0,
4119	s->mem_index, ot \| MO_LE);
4120	tcg_gen_sub_tl(s->T0, s->cc_srcT, s->T1);
4121	} else {
4122	tcg_gen_mov_tl(s->cc_srcT, s->T0);
4123	tcg_gen_sub_tl(s->T0, s->T0, s->T1);
4124	}
4125	prepare_update2_cc(decode, s, CC_OP_SUBB + ot);
4126	}
4127
4128	static void gen_SYSCALL(DisasContext s, X86DecodedInsn decode)
4129	{
4130	gen_update_cc_op(s);
4131	gen_update_eip_cur(s);
4132	gen_helper_syscall(tcg_env, cur_insn_len_i32(s));
4133	if (LMA(s)) {
4134	assume_cc_op(s, CC_OP_EFLAGS);
4135	}
4136
4137	/*
4138	* TF handling for the syscall insn is different. The TF bit is checked
4139	* after the syscall insn completes. This allows #DB to not be
4140	* generated after one has entered CPL0 if TF is set in FMASK.
4141	*/
4142	s->base.is_jmp = DISAS_EOB_RECHECK_TF;
4143	}
4144
4145	static void gen_SYSENTER(DisasContext s, X86DecodedInsn decode)
4146	{
4147	gen_helper_sysenter(tcg_env);
4148	s->base.is_jmp = DISAS_EOB_ONLY;
4149	}
4150
4151	static void gen_SYSEXIT(DisasContext s, X86DecodedInsn decode)
4152	{
4153	gen_helper_sysexit(tcg_env, tcg_constant_i32(s->dflag - 1));
4154	s->base.is_jmp = DISAS_EOB_ONLY;
4155	}
4156
4157	static void gen_SYSRET(DisasContext s, X86DecodedInsn decode)
4158	{
4159	gen_helper_sysret(tcg_env, tcg_constant_i32(s->dflag - 1));
4160	if (LMA(s)) {
4161	assume_cc_op(s, CC_OP_EFLAGS);
4162	}
4163
4164	/*
4165	* TF handling for the sysret insn is different. The TF bit is checked
4166	* after the sysret insn completes. This allows #DB to be
4167	* generated "as if" the syscall insn in userspace has just
4168	* completed.
4169	*/
4170	s->base.is_jmp = DISAS_EOB_RECHECK_TF;
4171	}
4172
4173	static void gen_TZCNT(DisasContext s, X86DecodedInsn decode)
4174	{
4175	MemOp ot = decode->op[0].ot;
4176
4177	/* C bit (cc_src) is defined related to the input. */
4178	decode->cc_src = tcg_temp_new();
4179	decode->cc_dst = s->T0;
4180	decode->cc_op = CC_OP_BMILGB + ot;
4181	tcg_gen_mov_tl(decode->cc_src, s->T0);
4182
4183	/* A zero input returns the operand size. */
4184	tcg_gen_ctzi_tl(s->T0, s->T0, 8 << ot);
4185	}
4186
4187	static void gen_UD(DisasContext s, X86DecodedInsn decode)
4188	{
4189	gen_illegal_opcode(s);
4190	}
4191
4192	static void gen_VAESIMC(DisasContext s, X86DecodedInsn decode)
4193	{
4194	assert(!s->vex_l);
4195	gen_helper_aesimc_xmm(tcg_env, OP_PTR0, OP_PTR2);
4196	}
4197
4198	/*
4199	* 00 = v*ps Vps, Hps, Wpd
4200	* 66 = v*pd Vpd, Hpd, Wps
4201	* f3 = v*ss Vss, Hss, Wps
4202	* f2 = v*sd Vsd, Hsd, Wps
4203	*/
4204	#define SSE_CMP(x) { \
4205	gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \
4206	gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \
4207	gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm}
4208	static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = {
4209	SSE_CMP(cmpeq),
4210	SSE_CMP(cmplt),
4211	SSE_CMP(cmple),
4212	SSE_CMP(cmpunord),
4213	SSE_CMP(cmpneq),
4214	SSE_CMP(cmpnlt),
4215	SSE_CMP(cmpnle),
4216	SSE_CMP(cmpord),
4217
4218	SSE_CMP(cmpequ),
4219	SSE_CMP(cmpnge),
4220	SSE_CMP(cmpngt),
4221	SSE_CMP(cmpfalse),
4222	SSE_CMP(cmpnequ),
4223	SSE_CMP(cmpge),
4224	SSE_CMP(cmpgt),
4225	SSE_CMP(cmptrue),
4226
4227	SSE_CMP(cmpeqs),
4228	SSE_CMP(cmpltq),
4229	SSE_CMP(cmpleq),
4230	SSE_CMP(cmpunords),
4231	SSE_CMP(cmpneqq),
4232	SSE_CMP(cmpnltq),
4233	SSE_CMP(cmpnleq),
4234	SSE_CMP(cmpords),
4235
4236	SSE_CMP(cmpequs),
4237	SSE_CMP(cmpngeq),
4238	SSE_CMP(cmpngtq),
4239	SSE_CMP(cmpfalses),
4240	SSE_CMP(cmpnequs),
4241	SSE_CMP(cmpgeq),
4242	SSE_CMP(cmpgtq),
4243	SSE_CMP(cmptrues),
4244	};
4245	#undef SSE_CMP
4246
4247	static void gen_VCMP(DisasContext s, X86DecodedInsn decode)
4248	{
4249	int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7);
4250	int b =
4251	s->prefix & PREFIX_REPZ ? 2 /* ss */ :
4252	s->prefix & PREFIX_REPNZ ? 3 /* sd */ :
4253	!!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2);
4254
4255	gen_helper_cmp_funcs[index][b](tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
4256	}
4257
4258	static void gen_VCOMI(DisasContext s, X86DecodedInsn decode)
4259	{
4260	SSEFunc_0_epp fn;
4261	fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss;
4262	fn(tcg_env, OP_PTR1, OP_PTR2);
4263	assume_cc_op(s, CC_OP_EFLAGS);
4264	}
4265
4266	static void gen_VCVTPD2PS(DisasContext s, X86DecodedInsn decode)
4267	{
4268	if (s->vex_l) {
4269	gen_helper_cvtpd2ps_ymm(tcg_env, OP_PTR0, OP_PTR2);
4270	} else {
4271	gen_helper_cvtpd2ps_xmm(tcg_env, OP_PTR0, OP_PTR2);
4272	}
4273	}
4274
4275	static void gen_VCVTPS2PD(DisasContext s, X86DecodedInsn decode)
4276	{
4277	if (s->vex_l) {
4278	gen_helper_cvtps2pd_ymm(tcg_env, OP_PTR0, OP_PTR2);
4279	} else {
4280	gen_helper_cvtps2pd_xmm(tcg_env, OP_PTR0, OP_PTR2);
4281	}
4282	}
4283
4284	static void gen_VCVTPS2PH(DisasContext s, X86DecodedInsn decode)
4285	{
4286	gen_unary_imm_fp_sse(s, decode,
4287	gen_helper_cvtps2ph_xmm,
4288	gen_helper_cvtps2ph_ymm);
4289	/*
4290	* VCVTPS2PH is the only instruction that performs an operation on a
4291	* register source and then stores into memory.
4292	*/
4293	if (decode->op[0].has_ea) {
4294	gen_store_sse(s, decode, decode->op[0].offset);
4295	}
4296	}
4297
4298	static void gen_VCVTSD2SS(DisasContext s, X86DecodedInsn decode)
4299	{
4300	gen_helper_cvtsd2ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
4301	}
4302
4303	static void gen_VCVTSS2SD(DisasContext s, X86DecodedInsn decode)
4304	{
4305	gen_helper_cvtss2sd(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2);
4306	}
4307
4308	static void gen_VCVTSI2Sx(DisasContext s, X86DecodedInsn decode)
4309	{
4310	int vec_len = vector_len(s, decode);
4311	TCGv_i32 in;
4312
4313	tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
4314
4315	#ifdef TARGET_X86_64
4316	MemOp ot = decode->op[2].ot;
4317	if (ot == MO_64) {
4318	if (s->prefix & PREFIX_REPNZ) {
4319	gen_helper_cvtsq2sd(tcg_env, OP_PTR0, s->T1);
4320	} else {
4321	gen_helper_cvtsq2ss(tcg_env, OP_PTR0, s->T1);
4322	}
4323	return;
4324	}
4325	in = tcg_temp_new_i32();
4326	tcg_gen_trunc_tl_i32(in, s->T1);
4327	#else
4328	in = s->T1;
4329	#endif
4330
4331	if (s->prefix & PREFIX_REPNZ) {
4332	gen_helper_cvtsi2sd(tcg_env, OP_PTR0, in);
4333	} else {
4334	gen_helper_cvtsi2ss(tcg_env, OP_PTR0, in);
4335	}
4336	}
4337
4338	static inline void gen_VCVTtSx2SI(DisasContext s, X86DecodedInsn decode,
4339	SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq,
4340	SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq)
4341	{
4342	TCGv_i32 out;
4343
4344	#ifdef TARGET_X86_64
4345	MemOp ot = decode->op[0].ot;
4346	if (ot == MO_64) {
4347	if (s->prefix & PREFIX_REPNZ) {
4348	sd2sq(s->T0, tcg_env, OP_PTR2);
4349	} else {
4350	ss2sq(s->T0, tcg_env, OP_PTR2);
4351	}
4352	return;
4353	}
4354
4355	out = tcg_temp_new_i32();
4356	#else
4357	out = s->T0;
4358	#endif
4359	if (s->prefix & PREFIX_REPNZ) {
4360	sd2si(out, tcg_env, OP_PTR2);
4361	} else {
4362	ss2si(out, tcg_env, OP_PTR2);
4363	}
4364	#ifdef TARGET_X86_64
4365	tcg_gen_extu_i32_tl(s->T0, out);
4366	#endif
4367	}
4368
4369	#ifndef TARGET_X86_64
4370	#define gen_helper_cvtss2sq NULL
4371	#define gen_helper_cvtsd2sq NULL
4372	#define gen_helper_cvttss2sq NULL
4373	#define gen_helper_cvttsd2sq NULL
4374	#endif
4375
4376	static void gen_VCVTSx2SI(DisasContext s, X86DecodedInsn decode)
4377	{
4378	gen_VCVTtSx2SI(s, decode,
4379	gen_helper_cvtss2si, gen_helper_cvtss2sq,
4380	gen_helper_cvtsd2si, gen_helper_cvtsd2sq);
4381	}
4382
4383	static void gen_VCVTTSx2SI(DisasContext s, X86DecodedInsn decode)
4384	{
4385	gen_VCVTtSx2SI(s, decode,
4386	gen_helper_cvttss2si, gen_helper_cvttss2sq,
4387	gen_helper_cvttsd2si, gen_helper_cvttsd2sq);
4388	}
4389
4390	static void gen_VEXTRACTx128(DisasContext s, X86DecodedInsn decode)
4391	{
4392	int mask = decode->immediate & 1;
4393	int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask);
4394	if (decode->op[0].has_ea) {
4395	/* VEX-only instruction, no alignment requirements. */
4396	gen_sto_env_A0(s, src_ofs, false);
4397	} else {
4398	tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16);
4399	}
4400	}
4401
4402	static void gen_VEXTRACTPS(DisasContext s, X86DecodedInsn decode)
4403	{
4404	gen_pextr(s, decode, MO_32);
4405	}
4406
4407	static void gen_vinsertps(DisasContext s, X86DecodedInsn decode, TCGv_i32 tmp)
4408	{
4409	int val = decode->immediate;
4410	int dest_word = (val >> 4) & 3;
4411	int new_mask = (val & 15) \| (1 << dest_word);
4412	int vec_len = 16;
4413
4414	assert(!s->vex_l);
4415
4416	if (new_mask == 15) {
4417	/* All zeroes except possibly for the inserted element */
4418	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
4419	} else if (decode->op[1].offset != decode->op[0].offset) {
4420	gen_store_sse(s, decode, decode->op[1].offset);
4421	}
4422
4423	if (new_mask != (val & 15)) {
4424	tcg_gen_st_i32(tmp, tcg_env,
4425	vector_elem_offset(&decode->op[0], MO_32, dest_word));
4426	}
4427
4428	if (new_mask != 15) {
4429	TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */
4430	int i;
4431	for (i = 0; i < 4; i++) {
4432	if ((val >> i) & 1) {
4433	tcg_gen_st_i32(zero, tcg_env,
4434	vector_elem_offset(&decode->op[0], MO_32, i));
4435	}
4436	}
4437	}
4438	}
4439
4440	static void gen_VINSERTPS_r(DisasContext s, X86DecodedInsn decode)
4441	{
4442	int val = decode->immediate;
4443	TCGv_i32 tmp = tcg_temp_new_i32();
4444
4445	tcg_gen_ld_i32(tmp, tcg_env,
4446	vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3));
4447	gen_vinsertps(s, decode, tmp);
4448	}
4449
4450	static void gen_VINSERTPS_m(DisasContext s, X86DecodedInsn decode)
4451	{
4452	TCGv_i32 tmp = tcg_temp_new_i32();
4453
4454	tcg_gen_qemu_ld_i32(tmp, s->A0, s->mem_index, MO_LEUL);
4455	gen_vinsertps(s, decode, tmp);
4456	}
4457
4458	static void gen_VINSERTx128(DisasContext s, X86DecodedInsn decode)
4459	{
4460	int mask = decode->immediate & 1;
4461	tcg_gen_gvec_mov(MO_64,
4462	decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)),
4463	decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16);
4464	tcg_gen_gvec_mov(MO_64,
4465	decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)),
4466	decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16);
4467	}
4468
4469	static inline void gen_maskmov(DisasContext s, X86DecodedInsn decode,
4470	SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm)
4471	{
4472	if (!s->vex_l) {
4473	xmm(tcg_env, OP_PTR2, OP_PTR1, s->A0);
4474	} else {
4475	ymm(tcg_env, OP_PTR2, OP_PTR1, s->A0);
4476	}
4477	}
4478
4479	static void gen_VMASKMOVPD_st(DisasContext s, X86DecodedInsn decode)
4480	{
4481	gen_maskmov(s, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm);
4482	}
4483
4484	static void gen_VMASKMOVPS_st(DisasContext s, X86DecodedInsn decode)
4485	{
4486	gen_maskmov(s, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm);
4487	}
4488
4489	static void gen_VMOVHPx_ld(DisasContext s, X86DecodedInsn decode)
4490	{
4491	TCGv_i64 t = tcg_temp_new_i64();
4492
4493	gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
4494	if (decode->op[0].offset != decode->op[1].offset) {
4495	tcg_gen_ld_i64(t, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
4496	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
4497	}
4498	}
4499
4500	static void gen_VMOVHPx_st(DisasContext s, X86DecodedInsn decode)
4501	{
4502	gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
4503	}
4504
4505	static void gen_VMOVHPx(DisasContext s, X86DecodedInsn decode)
4506	{
4507	TCGv_i64 t = tcg_temp_new_i64();
4508
4509	if (decode->op[0].offset != decode->op[2].offset) {
4510	tcg_gen_ld_i64(t, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
4511	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
4512	}
4513	if (decode->op[0].offset != decode->op[1].offset) {
4514	tcg_gen_ld_i64(t, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
4515	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
4516	}
4517	}
4518
4519	static void gen_VMOVHLPS(DisasContext s, X86DecodedInsn decode)
4520	{
4521	TCGv_i64 t = tcg_temp_new_i64();
4522
4523	tcg_gen_ld_i64(t, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
4524	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
4525	if (decode->op[0].offset != decode->op[1].offset) {
4526	tcg_gen_ld_i64(t, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1)));
4527	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
4528	}
4529	}
4530
4531	static void gen_VMOVLHPS(DisasContext s, X86DecodedInsn decode)
4532	{
4533	TCGv_i64 t = tcg_temp_new_i64();
4534
4535	tcg_gen_ld_i64(t, tcg_env, decode->op[2].offset);
4536	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
4537	if (decode->op[0].offset != decode->op[1].offset) {
4538	tcg_gen_ld_i64(t, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
4539	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
4540	}
4541	}
4542
4543	/*
4544	* Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx.
4545	* Use a gvec move to move everything above the bottom 64 bits.
4546	*/
4547
4548	static void gen_VMOVLPx(DisasContext s, X86DecodedInsn decode)
4549	{
4550	int vec_len = vector_len(s, decode);
4551	TCGv_i64 t = tcg_temp_new_i64();
4552
4553	tcg_gen_ld_i64(t, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0)));
4554	tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
4555	tcg_gen_st_i64(t, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
4556	}
4557
4558	static void gen_VMOVLPx_ld(DisasContext s, X86DecodedInsn decode)
4559	{
4560	int vec_len = vector_len(s, decode);
4561	TCGv_i64 t = tcg_temp_new_i64();
4562
4563	tcg_gen_qemu_ld_i64(t, s->A0, s->mem_index, MO_LEUQ);
4564	tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
4565	tcg_gen_st_i64(t, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0)));
4566	}
4567
4568	static void gen_VMOVLPx_st(DisasContext s, X86DecodedInsn decode)
4569	{
4570	TCGv_i64 t = tcg_temp_new_i64();
4571
4572	tcg_gen_ld_i64(t, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0)));
4573	tcg_gen_qemu_st_i64(t, s->A0, s->mem_index, MO_LEUQ);
4574	}
4575
4576	static void gen_VMOVSD_ld(DisasContext s, X86DecodedInsn decode)
4577	{
4578	TCGv_i64 zero = tcg_constant_i64(0);
4579	TCGv_i64 t = tcg_temp_new_i64();
4580
4581	tcg_gen_qemu_ld_i64(t, s->A0, s->mem_index, MO_LEUQ);
4582	tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1)));
4583	tcg_gen_st_i64(t, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0)));
4584	}
4585
4586	static void gen_VMOVSS(DisasContext s, X86DecodedInsn decode)
4587	{
4588	int vec_len = vector_len(s, decode);
4589	TCGv_i32 tmp = tcg_temp_new_i32();
4590
4591	tcg_gen_ld_i32(tmp, OP_PTR2, offsetof(ZMMReg, ZMM_L(0)));
4592	tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
4593	tcg_gen_st_i32(tmp, OP_PTR0, offsetof(ZMMReg, ZMM_L(0)));
4594	}
4595
4596	static void gen_VMOVSS_ld(DisasContext s, X86DecodedInsn decode)
4597	{
4598	int vec_len = vector_len(s, decode);
4599	TCGv_i32 tmp = tcg_temp_new_i32();
4600
4601	tcg_gen_qemu_ld_i32(tmp, s->A0, s->mem_index, MO_LEUL);
4602	tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0);
4603	tcg_gen_st_i32(tmp, OP_PTR0, offsetof(ZMMReg, ZMM_L(0)));
4604	}
4605
4606	static void gen_VMOVSS_st(DisasContext s, X86DecodedInsn decode)
4607	{
4608	TCGv_i32 tmp = tcg_temp_new_i32();
4609
4610	tcg_gen_ld_i32(tmp, OP_PTR2, offsetof(ZMMReg, ZMM_L(0)));
4611	tcg_gen_qemu_st_i32(tmp, s->A0, s->mem_index, MO_LEUL);
4612	}
4613
4614	static void gen_VPMASKMOV_st(DisasContext s, X86DecodedInsn decode)
4615	{
4616	if (s->vex_w) {
4617	gen_VMASKMOVPD_st(s, decode);
4618	} else {
4619	gen_VMASKMOVPS_st(s, decode);
4620	}
4621	}
4622
4623	static void gen_VPERMD(DisasContext s, X86DecodedInsn decode)
4624	{
4625	assert(s->vex_l);
4626	gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2);
4627	}
4628
4629	static void gen_VPERM2x128(DisasContext s, X86DecodedInsn decode)
4630	{
4631	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
4632	assert(s->vex_l);
4633	gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm);
4634	}
4635
4636	static void gen_VPHMINPOSUW(DisasContext s, X86DecodedInsn decode)
4637	{
4638	assert(!s->vex_l);
4639	gen_helper_phminposuw_xmm(tcg_env, OP_PTR0, OP_PTR2);
4640	}
4641
4642	static void gen_VROUNDSD(DisasContext s, X86DecodedInsn decode)
4643	{
4644	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
4645	assert(!s->vex_l);
4646	gen_helper_roundsd_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
4647	}
4648
4649	static void gen_VROUNDSS(DisasContext s, X86DecodedInsn decode)
4650	{
4651	TCGv_i32 imm = tcg_constant8u_i32(decode->immediate);
4652	assert(!s->vex_l);
4653	gen_helper_roundss_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
4654	}
4655
4656	static void gen_VSHUF(DisasContext s, X86DecodedInsn decode)
4657	{
4658	TCGv_i32 imm = tcg_constant_i32(decode->immediate);
4659	SSEFunc_0_pppi ps, pd, fn;
4660	ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm;
4661	pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm;
4662	fn = s->prefix & PREFIX_DATA ? pd : ps;
4663	fn(OP_PTR0, OP_PTR1, OP_PTR2, imm);
4664	}
4665
4666	static void gen_VUCOMI(DisasContext s, X86DecodedInsn decode)
4667	{
4668	SSEFunc_0_epp fn;
4669	fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss;
4670	fn(tcg_env, OP_PTR1, OP_PTR2);
4671	assume_cc_op(s, CC_OP_EFLAGS);
4672	}
4673
4674	static void gen_VZEROALL(DisasContext s, X86DecodedInsn decode)
4675	{
4676	TCGv_ptr ptr = tcg_temp_new_ptr();
4677
4678	tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_regs));
4679	gen_helper_memset(ptr, ptr, tcg_constant_i32(0),
4680	tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg)));
4681	}
4682
4683	static void gen_VZEROUPPER(DisasContext s, X86DecodedInsn decode)
4684	{
4685	int i;
4686
4687	for (i = 0; i < CPU_NB_REGS; i++) {
4688	int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1));
4689	tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0);
4690	}
4691	}
4692
4693	static void gen_WAIT(DisasContext s, X86DecodedInsn decode)
4694	{
4695	if ((s->flags & (HF_MP_MASK \| HF_TS_MASK)) == (HF_MP_MASK \| HF_TS_MASK)) {
4696	gen_NM_exception(s);
4697	} else {
4698	/* needs to be treated as I/O because of ferr_irq */
4699	translator_io_start(&s->base);
4700	gen_helper_fwait(tcg_env);
4701	}
4702	}
4703
4704	#ifndef CONFIG_USER_ONLY
4705	static void gen_WRMSR(DisasContext s, X86DecodedInsn decode)
4706	{
4707	gen_update_cc_op(s);
4708	gen_update_eip_cur(s);
4709	gen_helper_wrmsr(tcg_env);
4710	s->base.is_jmp = DISAS_EOB_NEXT;
4711	}
4712	#else
4713	#define gen_WRMSR gen_unreachable
4714	#endif
4715
4716	static void gen_WRxxBASE(DisasContext s, X86DecodedInsn decode)
4717	{
4718	TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS];
4719
4720	/* Preserve hflags bits by testing CR4 at runtime. */
4721	gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK));
4722	tcg_gen_mov_tl(base, s->T0);
4723	}
4724
4725	static void gen_XADD(DisasContext s, X86DecodedInsn decode)
4726	{
4727	MemOp ot = decode->op[1].ot;
4728
4729	decode->cc_dst = tcg_temp_new();
4730	decode->cc_src = s->T1;
4731	decode->cc_op = CC_OP_ADDB + ot;
4732
4733	if (s->prefix & PREFIX_LOCK) {
4734	tcg_gen_atomic_fetch_add_tl(s->T0, s->A0, s->T1, s->mem_index, ot \| MO_LE);
4735	tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1);
4736	} else {
4737	tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1);
4738	/*
4739	* NOTE: writing memory first is important for MMU exceptions,
4740	* but "new result" wins for XADD AX, AX.
4741	*/
4742	gen_writeback(s, decode, 0, decode->cc_dst);
4743	}
4744	if (decode->op[0].has_ea \|\| decode->op[2].n != decode->op[0].n) {
4745	gen_writeback(s, decode, 2, s->T0);
4746	}
4747	}
4748
4749	static void gen_XCHG(DisasContext s, X86DecodedInsn decode)
4750	{
4751	if (s->prefix & PREFIX_LOCK) {
4752	tcg_gen_atomic_xchg_tl(s->T0, s->A0, s->T1,
4753	s->mem_index, decode->op[0].ot \| MO_LE);
4754	/* now store old value into register operand */
4755	gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0);
4756	} else {
4757	/* move destination value into source operand, source preserved in T1 */
4758	gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0);
4759	tcg_gen_mov_tl(s->T0, s->T1);
4760	}
4761	}
4762
4763	static void gen_XLAT(DisasContext s, X86DecodedInsn decode)
4764	{
4765	/* AL is already zero-extended into s->T0. */
4766	tcg_gen_add_tl(s->A0, cpu_regs[R_EBX], s->T0);
4767	gen_lea_v_seg(s, s->A0, R_DS, s->override);
4768	gen_op_ld_v(s, MO_8, s->T0, s->A0);
4769	}
4770
4771	static void gen_XOR(DisasContext s, X86DecodedInsn decode)
4772	{
4773	/* special case XOR reg, reg */
4774	if (decode->op[1].unit == X86_OP_INT &&
4775	decode->op[2].unit == X86_OP_INT &&
4776	decode->op[1].n == decode->op[2].n) {
4777	tcg_gen_movi_tl(s->T0, 0);
4778	decode->cc_op = CC_OP_EFLAGS;
4779	decode->cc_src = tcg_constant_tl(CC_Z \| CC_P);
4780	} else {
4781	MemOp ot = decode->op[1].ot;
4782
4783	if (s->prefix & PREFIX_LOCK) {
4784	tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T1,
4785	s->mem_index, ot \| MO_LE);
4786	} else {
4787	tcg_gen_xor_tl(s->T0, s->T0, s->T1);
4788	}
4789	prepare_update1_cc(decode, s, CC_OP_LOGICB + ot);
4790	}
4791	}
4792
4793	static void gen_XRSTOR(DisasContext s, X86DecodedInsn decode)
4794	{
4795	TCGv_i64 features = tcg_temp_new_i64();
4796
4797	tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]);
4798	gen_helper_xrstor(tcg_env, s->A0, features);
4799	if (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_MPX) {
4800	/*
4801	* XRSTOR is how MPX is enabled, which changes how
4802	* we translate. Thus we need to end the TB.
4803	*/
4804	s->base.is_jmp = DISAS_EOB_NEXT;
4805	}
4806	}
4807
4808	static void gen_XSAVE(DisasContext s, X86DecodedInsn decode)
4809	{
4810	TCGv_i64 features = tcg_temp_new_i64();
4811
4812	tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]);
4813	gen_helper_xsave(tcg_env, s->A0, features);
4814	}
4815
4816	static void gen_XSAVEOPT(DisasContext s, X86DecodedInsn decode)
4817	{
4818	TCGv_i64 features = tcg_temp_new_i64();
4819
4820	tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]);
4821	gen_helper_xsave(tcg_env, s->A0, features);
4822	}