translate.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 3,607 lines 113 KB

Raw

1	/*
2	* i386 translation
3	*
4	* Copyright (c) 2003 Fabrice Bellard
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19	#include "qemu/osdep.h"
20
21	#include "qemu/host-utils.h"
22	#include "cpu.h"
23	#include "accel/tcg/cpu-mmu-index.h"
24	#include "exec/translation-block.h"
25	#include "tcg/tcg-op.h"
26	#include "tcg/tcg-op-gvec.h"
27	#include "exec/translator.h"
28	#include "exec/target_page.h"
29	#include "fpu/softfloat.h"
30
31	#include "exec/helper-proto.h"
32	#include "exec/helper-gen.h"
33	#include "helper-tcg.h"
34	#include "decode-new.h"
35
36	#include "exec/log.h"
37
38	#define HELPER_H "helper.h"
39	#include "exec/helper-info.c.inc"
40	#undef HELPER_H
41
42	/* Fixes for Windows namespace pollution. */
43	#undef IN
44	#undef OUT
45
46	#define PREFIX_REPZ 0x01
47	#define PREFIX_REPNZ 0x02
48	#define PREFIX_LOCK 0x04
49	#define PREFIX_DATA 0x08
50	#define PREFIX_ADR 0x10
51	#define PREFIX_VEX 0x20
52	#define PREFIX_REX 0x40
53
54	#ifdef TARGET_X86_64
55	# define ctztl ctz64
56	# define clztl clz64
57	#else
58	# define ctztl ctz32
59	# define clztl clz32
60	#endif
61
62	/* For a switch indexed by MODRM, match all memory operands for a given OP. */
63	#define CASE_MODRM_MEM_OP(OP) \
64	case (0 << 6) \| (OP << 3) \| 0 ... (0 << 6) \| (OP << 3) \| 7: \
65	case (1 << 6) \| (OP << 3) \| 0 ... (1 << 6) \| (OP << 3) \| 7: \
66	case (2 << 6) \| (OP << 3) \| 0 ... (2 << 6) \| (OP << 3) \| 7
67
68	#define CASE_MODRM_OP(OP) \
69	case (0 << 6) \| (OP << 3) \| 0 ... (0 << 6) \| (OP << 3) \| 7: \
70	case (1 << 6) \| (OP << 3) \| 0 ... (1 << 6) \| (OP << 3) \| 7: \
71	case (2 << 6) \| (OP << 3) \| 0 ... (2 << 6) \| (OP << 3) \| 7: \
72	case (3 << 6) \| (OP << 3) \| 0 ... (3 << 6) \| (OP << 3) \| 7
73
74	//#define MACRO_TEST 1
75
76	/* global register indexes */
77	static TCGv cpu_cc_dst, cpu_cc_src, cpu_cc_src2;
78	static TCGv cpu_eip;
79	static TCGv_i32 cpu_cc_op;
80	static TCGv cpu_regs[CPU_NB_REGS];
81	static TCGv cpu_seg_base[6];
82	static TCGv_i64 cpu_bndl[4];
83	static TCGv_i64 cpu_bndu[4];
84
85	typedef struct DisasContext {
86	DisasContextBase base;
87
88	target_ulong pc; /* pc = eip + cs_base */
89	target_ulong cs_base; /* base of CS segment */
90	target_ulong pc_save;
91
92	MemOp aflag;
93	MemOp dflag;
94
95	int8_t override; /* -1 if no override, else R_CS, R_DS, etc */
96	uint8_t prefix;
97
98	bool has_modrm;
99	uint8_t modrm;
100
101	#ifndef CONFIG_USER_ONLY
102	uint8_t cpl; /* code priv level */
103	uint8_t iopl; /* i/o priv level */
104	#endif
105	uint8_t vex_l; /* vex vector length */
106	uint8_t vex_v; /* vex vvvv register, without 1's complement. */
107	uint8_t popl_esp_hack; /* for correct popl with esp base handling */
108	uint8_t rip_offset; /* only used in x86_64, but left for simplicity */
109
110	#ifdef TARGET_X86_64
111	uint8_t rex_r;
112	uint8_t rex_x;
113	uint8_t rex_b;
114	#endif
115	bool vex_w; /* used by AVX even on 32-bit processors */
116	bool jmp_opt; /* use direct block chaining for direct jumps */
117	bool cc_op_dirty;
118
119	CCOp cc_op; /* current CC operation */
120	int mem_index; /* select memory access functions */
121	uint32_t flags; /* all execution flags */
122	int cpuid_features;
123	int cpuid_ext_features;
124	int cpuid_ext2_features;
125	int cpuid_ext3_features;
126	int cpuid_7_0_ebx_features;
127	int cpuid_7_0_ecx_features;
128	int cpuid_7_1_eax_features;
129	int cpuid_xsave_features;
130
131	/* TCG local temps */
132	TCGv cc_srcT;
133	TCGv A0;
134	TCGv T0;
135	TCGv T1;
136
137	sigjmp_buf jmpbuf;
138	TCGOp *prev_insn_start;
139	TCGOp *prev_insn_end;
140	} DisasContext;
141
142	/*
143	* Point EIP to next instruction before ending translation.
144	* For instructions that can change hflags.
145	*/
146	#define DISAS_EOB_NEXT DISAS_TARGET_0
147
148	/*
149	* Point EIP to next instruction and set HF_INHIBIT_IRQ if not
150	* already set. For instructions that activate interrupt shadow.
151	*/
152	#define DISAS_EOB_INHIBIT_IRQ DISAS_TARGET_1
153
154	/*
155	* Return to the main loop; EIP might have already been updated
156	* but even in that case do not use lookup_and_goto_ptr().
157	*/
158	#define DISAS_EOB_ONLY DISAS_TARGET_2
159
160	/*
161	* EIP has already been updated. For jumps that wish to use
162	* lookup_and_goto_ptr()
163	*/
164	#define DISAS_JUMP DISAS_TARGET_3
165
166	/*
167	* EIP has already been updated. Use updated value of
168	* EFLAGS.TF to determine singlestep trap (SYSCALL/SYSRET).
169	*/
170	#define DISAS_EOB_RECHECK_TF DISAS_TARGET_4
171
172	/* The environment in which user-only runs is constrained. */
173	#ifdef CONFIG_USER_ONLY
174	#define PE(S) true
175	#define CPL(S) 3
176	#define IOPL(S) 0
177	#define SVME(S) false
178	#define GUEST(S) false
179	#else
180	#define PE(S) (((S)->flags & HF_PE_MASK) != 0)
181	#define CPL(S) ((S)->cpl)
182	#define IOPL(S) ((S)->iopl)
183	#define SVME(S) (((S)->flags & HF_SVME_MASK) != 0)
184	#define GUEST(S) (((S)->flags & HF_GUEST_MASK) != 0)
185	#endif
186	#if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
187	#define VM86(S) false
188	#define CODE32(S) true
189	#define SS32(S) true
190	#define ADDSEG(S) false
191	#else
192	#define VM86(S) (((S)->flags & HF_VM_MASK) != 0)
193	#define CODE32(S) (((S)->flags & HF_CS32_MASK) != 0)
194	#define SS32(S) (((S)->flags & HF_SS32_MASK) != 0)
195	#define ADDSEG(S) (((S)->flags & HF_ADDSEG_MASK) != 0)
196	#endif
197	#if !defined(TARGET_X86_64)
198	#define CODE64(S) false
199	#elif defined(CONFIG_USER_ONLY)
200	#define CODE64(S) true
201	#else
202	#define CODE64(S) (((S)->flags & HF_CS64_MASK) != 0)
203	#endif
204	#if defined(CONFIG_USER_ONLY) \|\| defined(TARGET_X86_64)
205	#define LMA(S) (((S)->flags & HF_LMA_MASK) != 0)
206	#else
207	#define LMA(S) false
208	#endif
209
210	#ifdef TARGET_X86_64
211	#define REX_PREFIX(S) (((S)->prefix & (PREFIX_REX \| PREFIX_VEX)) != 0)
212	#define REX_W(S) ((S)->vex_w)
213	#define REX_R(S) ((S)->rex_r + 0)
214	#define REX_X(S) ((S)->rex_x + 0)
215	#define REX_B(S) ((S)->rex_b + 0)
216	#else
217	#define REX_PREFIX(S) false
218	#define REX_W(S) false
219	#define REX_R(S) 0
220	#define REX_X(S) 0
221	#define REX_B(S) 0
222	#endif
223
224	/*
225	* Many system-only helpers are not reachable for user-only.
226	* Define stub generators here, so that we need not either sprinkle
227	* ifdefs through the translator, nor provide the helper function.
228	*/
229	#define STUB_HELPER(NAME, ...) \
230	static inline void gen_helper_##NAME(__VA_ARGS__) \
231	{ qemu_build_not_reached(); }
232
233	#ifdef CONFIG_USER_ONLY
234	STUB_HELPER(clgi, TCGv_env env)
235	STUB_HELPER(flush_page, TCGv_env env, TCGv addr)
236	STUB_HELPER(inb, TCGv ret, TCGv_env env, TCGv_i32 port)
237	STUB_HELPER(inw, TCGv ret, TCGv_env env, TCGv_i32 port)
238	STUB_HELPER(inl, TCGv ret, TCGv_env env, TCGv_i32 port)
239	STUB_HELPER(monitor, TCGv_env env, TCGv addr)
240	STUB_HELPER(mwait, TCGv_env env, TCGv_i32 pc_ofs)
241	STUB_HELPER(outb, TCGv_env env, TCGv_i32 port, TCGv_i32 val)
242	STUB_HELPER(outw, TCGv_env env, TCGv_i32 port, TCGv_i32 val)
243	STUB_HELPER(outl, TCGv_env env, TCGv_i32 port, TCGv_i32 val)
244	STUB_HELPER(stgi, TCGv_env env)
245	STUB_HELPER(svm_check_intercept, TCGv_env env, TCGv_i32 type)
246	STUB_HELPER(vmload, TCGv_env env, TCGv_i32 aflag)
247	STUB_HELPER(vmmcall, TCGv_env env)
248	STUB_HELPER(vmrun, TCGv_env env, TCGv_i32 aflag, TCGv_i32 pc_ofs)
249	STUB_HELPER(vmsave, TCGv_env env, TCGv_i32 aflag)
250	STUB_HELPER(write_crN, TCGv_env env, TCGv_i32 reg, TCGv val)
251	#endif
252
253	static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num);
254	static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num);
255	static void gen_exception_gpf(DisasContext *s);
256
257	/* i386 shift ops */
258	enum {
259	OP_ROL,
260	OP_ROR,
261	OP_RCL,
262	OP_RCR,
263	OP_SHL,
264	OP_SHR,
265	OP_SHL1, /* undocumented */
266	OP_SAR = 7,
267	};
268
269	enum {
270	JCC_O,
271	JCC_B,
272	JCC_Z,
273	JCC_BE,
274	JCC_S,
275	JCC_P,
276	JCC_L,
277	JCC_LE,
278	};
279
280	enum {
281	USES_CC_DST = 1,
282	USES_CC_SRC = 2,
283	USES_CC_SRC2 = 4,
284	USES_CC_SRCT = 8,
285	};
286
287	/* Bit set if the global variable is live after setting CC_OP to X. */
288	static const uint8_t cc_op_live_[] = {
289	[CC_OP_DYNAMIC] = USES_CC_DST \| USES_CC_SRC \| USES_CC_SRC2,
290	[CC_OP_EFLAGS] = USES_CC_SRC,
291	[CC_OP_MULB ... CC_OP_MULQ] = USES_CC_DST \| USES_CC_SRC,
292	[CC_OP_ADDB ... CC_OP_ADDQ] = USES_CC_DST \| USES_CC_SRC,
293	[CC_OP_ADCB ... CC_OP_ADCQ] = USES_CC_DST \| USES_CC_SRC \| USES_CC_SRC2,
294	[CC_OP_SUBB ... CC_OP_SUBQ] = USES_CC_DST \| USES_CC_SRC \| USES_CC_SRCT,
295	[CC_OP_SBBB ... CC_OP_SBBQ] = USES_CC_DST \| USES_CC_SRC \| USES_CC_SRC2,
296	[CC_OP_LOGICB ... CC_OP_LOGICQ] = USES_CC_DST,
297	[CC_OP_INCB ... CC_OP_INCQ] = USES_CC_DST \| USES_CC_SRC,
298	[CC_OP_DECB ... CC_OP_DECQ] = USES_CC_DST \| USES_CC_SRC,
299	[CC_OP_SHLB ... CC_OP_SHLQ] = USES_CC_DST \| USES_CC_SRC,
300	[CC_OP_SARB ... CC_OP_SARQ] = USES_CC_DST \| USES_CC_SRC,
301	[CC_OP_BMILGB ... CC_OP_BMILGQ] = USES_CC_DST \| USES_CC_SRC,
302	[CC_OP_BLSIB ... CC_OP_BLSIQ] = USES_CC_DST \| USES_CC_SRC,
303	[CC_OP_ADCX] = USES_CC_DST \| USES_CC_SRC,
304	[CC_OP_ADOX] = USES_CC_SRC \| USES_CC_SRC2,
305	[CC_OP_ADCOX] = USES_CC_DST \| USES_CC_SRC \| USES_CC_SRC2,
306	[CC_OP_POPCNT] = USES_CC_DST,
307	[CC_OP_SBB_SELF] = USES_CC_DST,
308	};
309
310	static uint8_t cc_op_live(CCOp op)
311	{
312	uint8_t result;
313	assert(op >= 0 && op < ARRAY_SIZE(cc_op_live_));
314
315	/*
316	* Check that the array is fully populated. A zero entry would correspond
317	* to a fixed value of EFLAGS, which can be obtained with CC_OP_EFLAGS
318	* as well.
319	*/
320	result = cc_op_live_[op];
321	assert(result);
322	return result;
323	}
324
325	static void set_cc_op_1(DisasContext *s, CCOp op, bool dirty)
326	{
327	int dead;
328
329	if (s->cc_op == op) {
330	return;
331	}
332
333	/* Discard CC computation that will no longer be used. */
334	dead = cc_op_live(s->cc_op) & ~cc_op_live(op);
335	if (dead & USES_CC_DST) {
336	tcg_gen_discard_tl(cpu_cc_dst);
337	}
338	if (dead & USES_CC_SRC) {
339	tcg_gen_discard_tl(cpu_cc_src);
340	}
341	if (dead & USES_CC_SRC2) {
342	tcg_gen_discard_tl(cpu_cc_src2);
343	}
344	if (dead & USES_CC_SRCT) {
345	s->cc_srcT = NULL;
346	}
347
348	if (dirty && s->cc_op == CC_OP_DYNAMIC) {
349	tcg_gen_discard_i32(cpu_cc_op);
350	}
351	s->cc_op_dirty = dirty;
352	s->cc_op = op;
353	}
354
355	static void set_cc_op(DisasContext *s, CCOp op)
356	{
357	/*
358	* The DYNAMIC setting is translator only, everything else
359	* will be spilled later.
360	*/
361	set_cc_op_1(s, op, op != CC_OP_DYNAMIC);
362	}
363
364	static void assume_cc_op(DisasContext *s, CCOp op)
365	{
366	set_cc_op_1(s, op, false);
367	}
368
369	static void gen_update_cc_op(DisasContext *s)
370	{
371	if (s->cc_op_dirty) {
372	tcg_gen_movi_i32(cpu_cc_op, s->cc_op);
373	s->cc_op_dirty = false;
374	}
375	}
376
377	/* In instruction encodings for byte register accesses the
378	* register number usually indicates "low 8 bits of register N";
379	* however there are some special cases where N 4..7 indicates
380	* [AH, CH, DH, BH], ie "bits 15..8 of register N-4". Return
381	* true for this special case, false otherwise.
382	*/
383	static inline bool byte_reg_is_xH(DisasContext *s, int reg)
384	{
385	/* Any time the REX prefix is present, byte registers are uniform */
386	if (reg < 4 \|\| REX_PREFIX(s)) {
387	return false;
388	}
389	return true;
390	}
391
392	/* Select the size of a push/pop operation. */
393	static inline MemOp mo_pushpop(DisasContext *s, MemOp ot)
394	{
395	if (CODE64(s)) {
396	return ot == MO_16 ? MO_16 : MO_64;
397	} else {
398	return ot;
399	}
400	}
401
402	/* Select the size of the stack pointer. */
403	static inline MemOp mo_stacksize(DisasContext *s)
404	{
405	return CODE64(s) ? MO_64 : SS32(s) ? MO_32 : MO_16;
406	}
407
408	/* Compute the result of writing t0 to the OT-sized register REG.
409	*
410	* If DEST is NULL, store the result into the register and return the
411	* register's TCGv.
412	*
413	* If DEST is not NULL, store the result into DEST and return the
414	* register's TCGv.
415	*/
416	static TCGv gen_op_deposit_reg_v(DisasContext *s, MemOp ot, int reg, TCGv dest, TCGv t0)
417	{
418	switch(ot) {
419	case MO_8:
420	if (byte_reg_is_xH(s, reg)) {
421	dest = dest ? dest : cpu_regs[reg - 4];
422	tcg_gen_deposit_tl(dest, cpu_regs[reg - 4], t0, 8, 8);
423	return cpu_regs[reg - 4];
424	}
425	dest = dest ? dest : cpu_regs[reg];
426	tcg_gen_deposit_tl(dest, cpu_regs[reg], t0, 0, 8);
427	break;
428	case MO_16:
429	dest = dest ? dest : cpu_regs[reg];
430	tcg_gen_deposit_tl(dest, cpu_regs[reg], t0, 0, 16);
431	break;
432	case MO_32:
433	#ifdef TARGET_X86_64
434	dest = dest ? dest : cpu_regs[reg];
435	tcg_gen_ext32u_tl(dest, t0);
436	break;
437	case MO_64:
438	#endif
439	dest = dest ? dest : cpu_regs[reg];
440	tcg_gen_mov_tl(dest, t0);
441	break;
442	default:
443	g_assert_not_reached();
444	}
445	return cpu_regs[reg];
446	}
447
448	static void gen_op_mov_reg_v(DisasContext *s, MemOp ot, int reg, TCGv t0)
449	{
450	gen_op_deposit_reg_v(s, ot, reg, NULL, t0);
451	}
452
453	static inline
454	void gen_op_mov_v_reg(DisasContext *s, MemOp ot, TCGv t0, int reg)
455	{
456	if (ot == MO_8 && byte_reg_is_xH(s, reg)) {
457	tcg_gen_shri_tl(t0, cpu_regs[reg - 4], 8);
458	} else {
459	tcg_gen_mov_tl(t0, cpu_regs[reg]);
460	}
461	}
462
463	static void gen_add_A0_im(DisasContext *s, int val)
464	{
465	tcg_gen_addi_tl(s->A0, s->A0, val);
466	if (!CODE64(s)) {
467	tcg_gen_ext32u_tl(s->A0, s->A0);
468	}
469	}
470
471	static inline void gen_op_jmp_v(DisasContext *s, TCGv dest)
472	{
473	tcg_gen_mov_tl(cpu_eip, dest);
474	s->pc_save = -1;
475	}
476
477	static inline void gen_op_add_reg(DisasContext *s, MemOp size, int reg, TCGv val)
478	{
479	/* Using cpu_regs[reg] does not work for xH registers. */
480	assert(size >= MO_16);
481	if (size == MO_16) {
482	TCGv temp = tcg_temp_new();
483	tcg_gen_add_tl(temp, cpu_regs[reg], val);
484	gen_op_mov_reg_v(s, size, reg, temp);
485	} else {
486	tcg_gen_add_tl(cpu_regs[reg], cpu_regs[reg], val);
487	tcg_gen_ext_tl(cpu_regs[reg], cpu_regs[reg], size);
488	}
489	}
490
491	static inline
492	void gen_op_add_reg_im(DisasContext *s, MemOp size, int reg, int32_t val)
493	{
494	gen_op_add_reg(s, size, reg, tcg_constant_tl(val));
495	}
496
497	static inline void gen_op_ld_v(DisasContext *s, int idx, TCGv t0, TCGv a0)
498	{
499	tcg_gen_qemu_ld_tl(t0, a0, s->mem_index, idx \| MO_LE);
500	}
501
502	static inline void gen_op_st_v(DisasContext *s, int idx, TCGv t0, TCGv a0)
503	{
504	tcg_gen_qemu_st_tl(t0, a0, s->mem_index, idx \| MO_LE);
505	}
506
507	static void gen_update_eip_next(DisasContext *s)
508	{
509	assert(s->pc_save != -1);
510	if (tb_cflags(s->base.tb) & CF_PCREL) {
511	tcg_gen_addi_tl(cpu_eip, cpu_eip, s->pc - s->pc_save);
512	} else if (CODE64(s)) {
513	tcg_gen_movi_tl(cpu_eip, s->pc);
514	} else {
515	tcg_gen_movi_tl(cpu_eip, (uint32_t)(s->pc - s->cs_base));
516	}
517	s->pc_save = s->pc;
518	}
519
520	static void gen_update_eip_cur(DisasContext *s)
521	{
522	assert(s->pc_save != -1);
523	if (tb_cflags(s->base.tb) & CF_PCREL) {
524	tcg_gen_addi_tl(cpu_eip, cpu_eip, s->base.pc_next - s->pc_save);
525	} else if (CODE64(s)) {
526	tcg_gen_movi_tl(cpu_eip, s->base.pc_next);
527	} else {
528	tcg_gen_movi_tl(cpu_eip, (uint32_t)(s->base.pc_next - s->cs_base));
529	}
530	s->pc_save = s->base.pc_next;
531	}
532
533	static int cur_insn_len(DisasContext *s)
534	{
535	return s->pc - s->base.pc_next;
536	}
537
538	static TCGv_i32 cur_insn_len_i32(DisasContext *s)
539	{
540	return tcg_constant_i32(cur_insn_len(s));
541	}
542
543	static TCGv_i32 eip_next_i32(DisasContext *s)
544	{
545	assert(s->pc_save != -1);
546	/*
547	* This function has two users: lcall_real (always 16-bit mode), and
548	* iret_protected (16, 32, or 64-bit mode). IRET only uses the value
549	* when EFLAGS.NT is set, which is illegal in 64-bit mode, which is
550	* why passing a 32-bit value isn't broken. To avoid using this where
551	* we shouldn't, return -1 in 64-bit mode so that execution goes into
552	* the weeds quickly.
553	*/
554	if (CODE64(s)) {
555	return tcg_constant_i32(-1);
556	}
557	if (tb_cflags(s->base.tb) & CF_PCREL) {
558	TCGv_i32 ret = tcg_temp_new_i32();
559	tcg_gen_trunc_tl_i32(ret, cpu_eip);
560	tcg_gen_addi_i32(ret, ret, s->pc - s->pc_save);
561	return ret;
562	} else {
563	return tcg_constant_i32(s->pc - s->cs_base);
564	}
565	}
566
567	static TCGv eip_next_tl(DisasContext *s)
568	{
569	assert(s->pc_save != -1);
570	if (tb_cflags(s->base.tb) & CF_PCREL) {
571	TCGv ret = tcg_temp_new();
572	tcg_gen_addi_tl(ret, cpu_eip, s->pc - s->pc_save);
573	return ret;
574	} else if (CODE64(s)) {
575	return tcg_constant_tl(s->pc);
576	} else {
577	return tcg_constant_tl((uint32_t)(s->pc - s->cs_base));
578	}
579	}
580
581	static TCGv eip_cur_tl(DisasContext *s)
582	{
583	assert(s->pc_save != -1);
584	if (tb_cflags(s->base.tb) & CF_PCREL) {
585	TCGv ret = tcg_temp_new();
586	tcg_gen_addi_tl(ret, cpu_eip, s->base.pc_next - s->pc_save);
587	return ret;
588	} else if (CODE64(s)) {
589	return tcg_constant_tl(s->base.pc_next);
590	} else {
591	return tcg_constant_tl((uint32_t)(s->base.pc_next - s->cs_base));
592	}
593	}
594
595	/* Compute SEG:REG into DEST. SEG is selected from the override segment
596	(OVR_SEG) and the default segment (DEF_SEG). OVR_SEG may be -1 to
597	indicate no override. */
598	static void gen_lea_v_seg_dest(DisasContext *s, MemOp aflag, TCGv dest, TCGv a0,
599	int def_seg, int ovr_seg)
600	{
601	int easize;
602	bool has_base;
603
604	if (ovr_seg < 0) {
605	ovr_seg = def_seg;
606	}
607
608	has_base = ovr_seg >= R_FS \|\| (ovr_seg >= 0 && ADDSEG(s));
609	easize = CODE64(s) ? MO_64 : MO_32;
610
611	if (has_base) {
612	if (aflag < easize) {
613	/* Truncate before summing base. */
614	tcg_gen_ext_tl(dest, a0, aflag);
615	a0 = dest;
616	}
617	tcg_gen_add_tl(dest, a0, cpu_seg_base[ovr_seg]);
618	a0 = dest;
619	} else {
620	/* Possibly one extension, but that's it. */
621	easize = aflag;
622	}
623
624	tcg_gen_ext_tl(dest, a0, easize);
625	}
626
627	static void gen_lea_v_seg(DisasContext *s, TCGv a0,
628	int def_seg, int ovr_seg)
629	{
630	gen_lea_v_seg_dest(s, s->aflag, s->A0, a0, def_seg, ovr_seg);
631	}
632
633	static inline void gen_string_movl_A0_ESI(DisasContext *s)
634	{
635	gen_lea_v_seg(s, cpu_regs[R_ESI], R_DS, s->override);
636	}
637
638	static inline void gen_string_movl_A0_EDI(DisasContext *s)
639	{
640	gen_lea_v_seg(s, cpu_regs[R_EDI], R_ES, -1);
641	}
642
643	static void gen_op_j_ecx(DisasContext s, TCGCond cond, TCGLabel label1)
644	{
645	TCGv cmpval;
646	if (s->aflag == MO_TL) {
647	cmpval = cpu_regs[R_ECX];
648	} else {
649	cmpval = tcg_temp_new();
650	tcg_gen_ext_tl(cmpval, cpu_regs[R_ECX], s->aflag);
651	}
652
653	tcg_gen_brcondi_tl(cond, cmpval, 0, label1);
654	}
655
656	static inline void gen_op_jz_ecx(DisasContext s, TCGLabel label1)
657	{
658	gen_op_j_ecx(s, TCG_COND_EQ, label1);
659	}
660
661	static inline void gen_op_jnz_ecx(DisasContext s, TCGLabel label1)
662	{
663	gen_op_j_ecx(s, TCG_COND_NE, label1);
664	}
665
666	static void gen_set_hflag(DisasContext *s, uint32_t mask)
667	{
668	if ((s->flags & mask) == 0) {
669	TCGv_i32 t = tcg_temp_new_i32();
670	tcg_gen_ld_i32(t, tcg_env, offsetof(CPUX86State, hflags));
671	tcg_gen_ori_i32(t, t, mask);
672	tcg_gen_st_i32(t, tcg_env, offsetof(CPUX86State, hflags));
673	s->flags \|= mask;
674	}
675	}
676
677	static void gen_reset_hflag(DisasContext *s, uint32_t mask)
678	{
679	if (s->flags & mask) {
680	TCGv_i32 t = tcg_temp_new_i32();
681	tcg_gen_ld_i32(t, tcg_env, offsetof(CPUX86State, hflags));
682	tcg_gen_andi_i32(t, t, ~mask);
683	tcg_gen_st_i32(t, tcg_env, offsetof(CPUX86State, hflags));
684	s->flags &= ~mask;
685	}
686	}
687
688	static void gen_set_eflags(DisasContext *s, target_ulong mask)
689	{
690	TCGv t = tcg_temp_new();
691
692	tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, eflags));
693	tcg_gen_ori_tl(t, t, mask);
694	tcg_gen_st_tl(t, tcg_env, offsetof(CPUX86State, eflags));
695	}
696
697	static void gen_reset_eflags(DisasContext *s, target_ulong mask)
698	{
699	TCGv t = tcg_temp_new();
700
701	tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, eflags));
702	tcg_gen_andi_tl(t, t, ~mask);
703	tcg_gen_st_tl(t, tcg_env, offsetof(CPUX86State, eflags));
704	}
705
706	static void gen_helper_in_func(MemOp ot, TCGv v, TCGv_i32 n)
707	{
708	switch (ot) {
709	case MO_8:
710	gen_helper_inb(v, tcg_env, n);
711	break;
712	case MO_16:
713	gen_helper_inw(v, tcg_env, n);
714	break;
715	case MO_32:
716	gen_helper_inl(v, tcg_env, n);
717	break;
718	default:
719	g_assert_not_reached();
720	}
721	}
722
723	static void gen_helper_out_func(MemOp ot, TCGv_i32 v, TCGv_i32 n)
724	{
725	switch (ot) {
726	case MO_8:
727	gen_helper_outb(tcg_env, v, n);
728	break;
729	case MO_16:
730	gen_helper_outw(tcg_env, v, n);
731	break;
732	case MO_32:
733	gen_helper_outl(tcg_env, v, n);
734	break;
735	default:
736	g_assert_not_reached();
737	}
738	}
739
740	/*
741	* Validate that access to [port, port + 1<<ot) is allowed.
742	* Raise #GP, or VMM exit if not.
743	*/
744	static bool gen_check_io(DisasContext *s, MemOp ot, TCGv_i32 port,
745	uint32_t svm_flags)
746	{
747	#ifdef CONFIG_USER_ONLY
748	/*
749	* We do not implement the ioperm(2) syscall, so the TSS check
750	* will always fail.
751	*/
752	gen_exception_gpf(s);
753	return false;
754	#else
755	if (PE(s) && (CPL(s) > IOPL(s) \|\| VM86(s))) {
756	gen_helper_check_io(tcg_env, port, tcg_constant_i32(1 << ot));
757	}
758	if (GUEST(s)) {
759	gen_update_cc_op(s);
760	gen_update_eip_cur(s);
761	if (s->prefix & (PREFIX_REPZ \| PREFIX_REPNZ)) {
762	svm_flags \|= SVM_IOIO_REP_MASK;
763	}
764	svm_flags \|= 1 << (SVM_IOIO_SIZE_SHIFT + ot);
765	gen_helper_svm_check_io(tcg_env, port,
766	tcg_constant_i32(svm_flags),
767	cur_insn_len_i32(s));
768	}
769	return true;
770	#endif
771	}
772
773	static void gen_movs(DisasContext *s, MemOp ot, TCGv dshift)
774	{
775	gen_string_movl_A0_ESI(s);
776	gen_op_ld_v(s, ot, s->T0, s->A0);
777	gen_string_movl_A0_EDI(s);
778	gen_op_st_v(s, ot, s->T0, s->A0);
779
780	gen_op_add_reg(s, s->aflag, R_ESI, dshift);
781	gen_op_add_reg(s, s->aflag, R_EDI, dshift);
782	}
783
784	/* compute all eflags to reg */
785	static void gen_mov_eflags(DisasContext *s, TCGv reg)
786	{
787	TCGv dst, src1, src2;
788	TCGv_i32 cc_op;
789	int live, dead;
790
791	if (s->cc_op == CC_OP_EFLAGS) {
792	tcg_gen_mov_tl(reg, cpu_cc_src);
793	return;
794	}
795
796	dst = cpu_cc_dst;
797	src1 = cpu_cc_src;
798	src2 = cpu_cc_src2;
799
800	/* Take care to not read values that are not live. */
801	live = cc_op_live(s->cc_op) & ~USES_CC_SRCT;
802	dead = live ^ (USES_CC_DST \| USES_CC_SRC \| USES_CC_SRC2);
803	if (dead) {
804	TCGv zero = tcg_constant_tl(0);
805	if (dead & USES_CC_DST) {
806	dst = zero;
807	}
808	if (dead & USES_CC_SRC) {
809	src1 = zero;
810	}
811	if (dead & USES_CC_SRC2) {
812	src2 = zero;
813	}
814	}
815
816	if (s->cc_op != CC_OP_DYNAMIC) {
817	cc_op = tcg_constant_i32(s->cc_op);
818	} else {
819	cc_op = cpu_cc_op;
820	}
821	gen_helper_cc_compute_all(reg, dst, src1, src2, cc_op);
822	}
823
824	/* compute all eflags to cc_src */
825	static void gen_compute_eflags(DisasContext *s)
826	{
827	gen_mov_eflags(s, cpu_cc_src);
828	set_cc_op(s, CC_OP_EFLAGS);
829	}
830
831	typedef enum {
832	/* imm is valid and reg2 is a constant */
833	CC_PREPARE_IMM,
834	/* imm is invalid */
835	CC_PREPARE_REG,
836	/* imm and reg2 are 0, and reg is known to be 0/1 */
837	CC_PREPARE_DIRECT,
838	} CCPrepareRHS;
839
840	typedef struct CCPrepare {
841	TCGCond cond;
842	TCGv reg;
843	TCGv reg2;
844	target_ulong imm;
845	CCPrepareRHS rhs_type;
846	} CCPrepare;
847
848	static CCPrepare gen_prepare_sign_nz(TCGv src, MemOp size)
849	{
850	if (size == MO_TL) {
851	return (CCPrepare) { .cond = TCG_COND_LT, .reg = src };
852	} else {
853	return (CCPrepare) { .cond = TCG_COND_TSTNE, .reg = src,
854	.imm = 1ull << ((8 << size) - 1) };
855	}
856	}
857
858	static CCPrepare gen_prepare_val_nz(TCGv src, MemOp size, bool eqz)
859	{
860	if (size == MO_TL) {
861	return (CCPrepare) { .cond = eqz ? TCG_COND_EQ : TCG_COND_NE,
862	.reg = src };
863	} else {
864	return (CCPrepare) { .cond = eqz ? TCG_COND_TSTEQ : TCG_COND_TSTNE,
865	.imm = MAKE_64BIT_MASK(0, 8 << size),
866	.reg = src };
867	}
868	}
869
870	/* compute eflags.C, trying to store it in reg if not NULL */
871	static CCPrepare gen_prepare_eflags_c(DisasContext *s, TCGv reg)
872	{
873	MemOp size;
874
875	switch (s->cc_op) {
876	case CC_OP_SUBB ... CC_OP_SUBQ:
877	/* (DATA_TYPE)CC_SRCT < (DATA_TYPE)CC_SRC */
878	size = s->cc_op - CC_OP_SUBB;
879	tcg_gen_ext_tl(s->cc_srcT, s->cc_srcT, size);
880	tcg_gen_ext_tl(cpu_cc_src, cpu_cc_src, size);
881	return (CCPrepare) { .cond = TCG_COND_LTU, .reg = s->cc_srcT,
882	.reg2 = cpu_cc_src, .rhs_type = CC_PREPARE_REG };
883
884	case CC_OP_ADDB ... CC_OP_ADDQ:
885	/* (DATA_TYPE)CC_DST < (DATA_TYPE)CC_SRC */
886	size = cc_op_size(s->cc_op);
887	tcg_gen_ext_tl(cpu_cc_dst, cpu_cc_dst, size);
888	tcg_gen_ext_tl(cpu_cc_src, cpu_cc_src, size);
889	return (CCPrepare) { .cond = TCG_COND_LTU, .reg = cpu_cc_dst,
890	.reg2 = cpu_cc_src, .rhs_type = CC_PREPARE_REG };
891
892	case CC_OP_LOGICB ... CC_OP_LOGICQ:
893	case CC_OP_POPCNT:
894	return (CCPrepare) { .cond = TCG_COND_NEVER };
895
896	case CC_OP_INCB ... CC_OP_INCQ:
897	case CC_OP_DECB ... CC_OP_DECQ:
898	return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src,
899	.rhs_type = CC_PREPARE_DIRECT };
900
901	case CC_OP_SHLB ... CC_OP_SHLQ:
902	/* (CC_SRC >> (DATA_BITS - 1)) & 1 */
903	size = cc_op_size(s->cc_op);
904	return gen_prepare_sign_nz(cpu_cc_src, size);
905
906	case CC_OP_MULB ... CC_OP_MULQ:
907	return (CCPrepare) { .cond = TCG_COND_NE,
908	.reg = cpu_cc_src };
909
910	case CC_OP_BMILGB ... CC_OP_BMILGQ:
911	size = cc_op_size(s->cc_op);
912	return gen_prepare_val_nz(cpu_cc_src, size, true);
913
914	case CC_OP_BLSIB ... CC_OP_BLSIQ:
915	size = cc_op_size(s->cc_op);
916	return gen_prepare_val_nz(cpu_cc_src, size, false);
917
918	case CC_OP_SBB_SELF:
919	return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_dst };
920
921	case CC_OP_ADCX:
922	case CC_OP_ADCOX:
923	return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_dst,
924	.rhs_type = CC_PREPARE_DIRECT };
925
926	case CC_OP_EFLAGS:
927	case CC_OP_SARB ... CC_OP_SARQ:
928	/* CC_SRC & 1 */
929	return (CCPrepare) { .cond = TCG_COND_TSTNE,
930	.reg = cpu_cc_src, .imm = CC_C };
931
932	default:
933	/* The need to compute only C from CC_OP_DYNAMIC is important
934	in efficiently implementing e.g. INC at the start of a TB. */
935	gen_update_cc_op(s);
936	if (!reg) {
937	reg = tcg_temp_new();
938	}
939	gen_helper_cc_compute_c(reg, cpu_cc_dst, cpu_cc_src,
940	cpu_cc_src2, cpu_cc_op);
941	return (CCPrepare) { .cond = TCG_COND_NE, .reg = reg,
942	.rhs_type = CC_PREPARE_DIRECT };
943	}
944	}
945
946	/* compute eflags.P, trying to store it in reg if not NULL */
947	static CCPrepare gen_prepare_eflags_p(DisasContext *s, TCGv reg)
948	{
949	gen_compute_eflags(s);
950	return (CCPrepare) { .cond = TCG_COND_TSTNE, .reg = cpu_cc_src,
951	.imm = CC_P };
952	}
953
954	/* compute eflags.S, trying to store it in reg if not NULL */
955	static CCPrepare gen_prepare_eflags_s(DisasContext *s, TCGv reg)
956	{
957	switch (s->cc_op) {
958	case CC_OP_DYNAMIC:
959	gen_compute_eflags(s);
960	/* FALLTHRU */
961	case CC_OP_EFLAGS:
962	case CC_OP_ADCX:
963	case CC_OP_ADOX:
964	case CC_OP_ADCOX:
965	return (CCPrepare) { .cond = TCG_COND_TSTNE, .reg = cpu_cc_src,
966	.imm = CC_S };
967	case CC_OP_POPCNT:
968	return (CCPrepare) { .cond = TCG_COND_NEVER };
969	default:
970	return gen_prepare_sign_nz(cpu_cc_dst, cc_op_size(s->cc_op));
971	}
972	}
973
974	/* compute eflags.O, trying to store it in reg if not NULL */
975	static CCPrepare gen_prepare_eflags_o(DisasContext *s, TCGv reg)
976	{
977	switch (s->cc_op) {
978	case CC_OP_ADOX:
979	case CC_OP_ADCOX:
980	return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src2,
981	.rhs_type = CC_PREPARE_DIRECT };
982	case CC_OP_SBB_SELF:
983	case CC_OP_LOGICB ... CC_OP_LOGICQ:
984	case CC_OP_POPCNT:
985	return (CCPrepare) { .cond = TCG_COND_NEVER };
986	case CC_OP_MULB ... CC_OP_MULQ:
987	return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src };
988	default:
989	gen_compute_eflags(s);
990	return (CCPrepare) { .cond = TCG_COND_TSTNE, .reg = cpu_cc_src,
991	.imm = CC_O };
992	}
993	}
994
995	/* compute eflags.Z, trying to store it in reg if not NULL */
996	static CCPrepare gen_prepare_eflags_z(DisasContext *s, TCGv reg)
997	{
998	switch (s->cc_op) {
999	case CC_OP_EFLAGS:
1000	case CC_OP_ADCX:
1001	case CC_OP_ADOX:
1002	case CC_OP_ADCOX:
1003	return (CCPrepare) { .cond = TCG_COND_TSTNE, .reg = cpu_cc_src,
1004	.imm = CC_Z };
1005	case CC_OP_DYNAMIC:
1006	gen_update_cc_op(s);
1007	if (!reg) {
1008	reg = tcg_temp_new();
1009	}
1010	gen_helper_cc_compute_nz(reg, cpu_cc_dst, cpu_cc_src, cpu_cc_op);
1011	return (CCPrepare) { .cond = TCG_COND_EQ, .reg = reg, .imm = 0 };
1012	case CC_OP_POPCNT:
1013	return (CCPrepare) { .cond = TCG_COND_EQ, .reg = cpu_cc_dst };
1014	default:
1015	{
1016	MemOp size = cc_op_size(s->cc_op);
1017	return gen_prepare_val_nz(cpu_cc_dst, size, true);
1018	}
1019	}
1020	}
1021
1022	/* return how to compute jump opcode 'b'. 'reg' can be clobbered
1023	* if needed; it may be used for CCPrepare.reg if that will
1024	* provide more freedom in the translation of a subsequent setcond. */
1025	static CCPrepare gen_prepare_cc(DisasContext *s, int b, TCGv reg)
1026	{
1027	int inv, jcc_op, cond;
1028	MemOp size;
1029	CCPrepare cc;
1030
1031	inv = b & 1;
1032	jcc_op = (b >> 1) & 7;
1033
1034	switch (s->cc_op) {
1035	case CC_OP_SUBB ... CC_OP_SUBQ:
1036	/* We optimize relational operators for the cmp/jcc case. */
1037	size = cc_op_size(s->cc_op);
1038	switch (jcc_op) {
1039	case JCC_BE:
1040	tcg_gen_ext_tl(s->cc_srcT, s->cc_srcT, size);
1041	tcg_gen_ext_tl(cpu_cc_src, cpu_cc_src, size);
1042	cc = (CCPrepare) { .cond = TCG_COND_LEU, .reg = s->cc_srcT,
1043	.reg2 = cpu_cc_src, .rhs_type = CC_PREPARE_REG };
1044	break;
1045	case JCC_L:
1046	cond = TCG_COND_LT;
1047	goto fast_jcc_l;
1048	case JCC_LE:
1049	cond = TCG_COND_LE;
1050	fast_jcc_l:
1051	tcg_gen_ext_tl(s->cc_srcT, s->cc_srcT, size \| MO_SIGN);
1052	tcg_gen_ext_tl(cpu_cc_src, cpu_cc_src, size \| MO_SIGN);
1053	cc = (CCPrepare) { .cond = cond, .reg = s->cc_srcT,
1054	.reg2 = cpu_cc_src, .rhs_type = CC_PREPARE_REG };
1055	break;
1056
1057	default:
1058	goto slow_jcc;
1059	}
1060	break;
1061
1062	case CC_OP_SBB_SELF:
1063	/* checking for nonzero is usually the most efficient */
1064	if (jcc_op == JCC_L \|\| jcc_op == JCC_B \|\| jcc_op == JCC_S) {
1065	jcc_op = JCC_Z;
1066	inv = !inv;
1067	}
1068	goto slow_jcc;
1069
1070	case CC_OP_LOGICB ... CC_OP_LOGICQ:
1071	/* Mostly used for test+jump */
1072	size = s->cc_op - CC_OP_LOGICB;
1073	switch (jcc_op) {
1074	case JCC_BE:
1075	/* CF = 0, becomes jz/je */
1076	jcc_op = JCC_Z;
1077	goto slow_jcc;
1078	case JCC_L:
1079	/* OF = 0, becomes js/jns */
1080	jcc_op = JCC_S;
1081	goto slow_jcc;
1082	case JCC_LE:
1083	/* SF or ZF, becomes signed <= 0 */
1084	tcg_gen_ext_tl(cpu_cc_dst, cpu_cc_dst, size \| MO_SIGN);
1085	cc = (CCPrepare) { .cond = TCG_COND_LE, .reg = cpu_cc_dst };
1086	break;
1087	default:
1088	goto slow_jcc;
1089	}
1090	break;
1091
1092	default:
1093	slow_jcc:
1094	/* This actually generates good code for JC, JZ and JS. */
1095	switch (jcc_op) {
1096	case JCC_O:
1097	cc = gen_prepare_eflags_o(s, reg);
1098	break;
1099	case JCC_B:
1100	cc = gen_prepare_eflags_c(s, reg);
1101	break;
1102	case JCC_Z:
1103	cc = gen_prepare_eflags_z(s, reg);
1104	break;
1105	case JCC_BE:
1106	gen_compute_eflags(s);
1107	cc = (CCPrepare) { .cond = TCG_COND_TSTNE, .reg = cpu_cc_src,
1108	.imm = CC_Z \| CC_C };
1109	break;
1110	case JCC_S:
1111	cc = gen_prepare_eflags_s(s, reg);
1112	break;
1113	case JCC_P:
1114	cc = gen_prepare_eflags_p(s, reg);
1115	break;
1116	case JCC_L:
1117	gen_compute_eflags(s);
1118	if (!reg \|\| reg == cpu_cc_src) {
1119	reg = tcg_temp_new();
1120	}
1121	tcg_gen_addi_tl(reg, cpu_cc_src, CC_O - CC_S);
1122	cc = (CCPrepare) { .cond = TCG_COND_TSTNE, .reg = reg,
1123	.imm = CC_O };
1124	break;
1125	default:
1126	case JCC_LE:
1127	gen_compute_eflags(s);
1128	if (!reg \|\| reg == cpu_cc_src) {
1129	reg = tcg_temp_new();
1130	}
1131	tcg_gen_addi_tl(reg, cpu_cc_src, CC_O - CC_S);
1132	cc = (CCPrepare) { .cond = TCG_COND_TSTNE, .reg = reg,
1133	.imm = CC_O \| CC_Z };
1134	break;
1135	}
1136	break;
1137	}
1138
1139	if (cc.rhs_type != CC_PREPARE_REG) {
1140	cc.reg2 = tcg_constant_tl(cc.imm);
1141	}
1142	if (inv) {
1143	cc.cond = tcg_invert_cond(cc.cond);
1144	}
1145	return cc;
1146	}
1147
1148	static void gen_neg_setcc(DisasContext *s, int b, TCGv reg)
1149	{
1150	CCPrepare cc = gen_prepare_cc(s, b, reg);
1151
1152	if (cc.rhs_type == CC_PREPARE_DIRECT) {
1153	if (cc.cond == TCG_COND_EQ) {
1154	tcg_gen_addi_tl(reg, cc.reg, -1);
1155	} else {
1156	tcg_gen_neg_tl(reg, cc.reg);
1157	}
1158	return;
1159	}
1160
1161	tcg_gen_negsetcond_tl(cc.cond, reg, cc.reg, cc.reg2);
1162	}
1163
1164	static void gen_setcc(DisasContext *s, int b, TCGv reg)
1165	{
1166	CCPrepare cc = gen_prepare_cc(s, b, reg);
1167
1168	if (cc.rhs_type == CC_PREPARE_DIRECT) {
1169	if (cc.cond == TCG_COND_EQ) {
1170	tcg_gen_xori_tl(reg, cc.reg, 1);
1171	} else {
1172	tcg_gen_mov_tl(reg, cc.reg);
1173	}
1174	return;
1175	}
1176
1177	tcg_gen_setcond_tl(cc.cond, reg, cc.reg, cc.reg2);
1178	}
1179
1180	static inline void gen_compute_eflags_c(DisasContext *s, TCGv reg)
1181	{
1182	gen_setcc(s, JCC_B << 1, reg);
1183	}
1184
1185	/* generate a conditional jump to label 'l1' according to jump opcode
1186	value 'b'. In the fast case, T0 is guaranteed not to be used. */
1187	static inline void gen_jcc_noeob(DisasContext s, int b, TCGLabel l1)
1188	{
1189	CCPrepare cc = gen_prepare_cc(s, b, NULL);
1190
1191	tcg_gen_brcond_tl(cc.cond, cc.reg, cc.reg2, l1);
1192	}
1193
1194	/* Generate a conditional jump to label 'l1' according to jump opcode
1195	value 'b'. In the fast case, T0 is guaranteed not to be used.
1196	One or both of the branches will call gen_jmp_rel, so ensure
1197	cc_op is clean. */
1198	static inline void gen_jcc(DisasContext s, int b, TCGLabel l1)
1199	{
1200	CCPrepare cc = gen_prepare_cc(s, b, NULL);
1201
1202	/*
1203	* Note that this must be _after_ gen_prepare_cc, because it can change
1204	* the cc_op to CC_OP_EFLAGS (because it's CC_OP_DYNAMIC or because
1205	* it's cheaper to just compute the flags)!
1206	*/
1207	gen_update_cc_op(s);
1208	tcg_gen_brcond_tl(cc.cond, cc.reg, cc.reg2, l1);
1209	}
1210
1211	static void gen_stos(DisasContext *s, MemOp ot, TCGv dshift)
1212	{
1213	gen_string_movl_A0_EDI(s);
1214	gen_op_st_v(s, ot, s->T0, s->A0);
1215	gen_op_add_reg(s, s->aflag, R_EDI, dshift);
1216	}
1217
1218	static void gen_lods(DisasContext *s, MemOp ot, TCGv dshift)
1219	{
1220	gen_string_movl_A0_ESI(s);
1221	gen_op_ld_v(s, ot, s->T0, s->A0);
1222	gen_op_mov_reg_v(s, ot, R_EAX, s->T0);
1223	gen_op_add_reg(s, s->aflag, R_ESI, dshift);
1224	}
1225
1226	static void gen_scas(DisasContext *s, MemOp ot, TCGv dshift)
1227	{
1228	s->cc_srcT = tcg_temp_new();
1229	gen_string_movl_A0_EDI(s);
1230	gen_op_ld_v(s, ot, s->T1, s->A0);
1231	tcg_gen_mov_tl(cpu_cc_src, s->T1);
1232	tcg_gen_mov_tl(s->cc_srcT, s->T0);
1233	tcg_gen_sub_tl(cpu_cc_dst, s->T0, s->T1);
1234	set_cc_op(s, CC_OP_SUBB + ot);
1235
1236	gen_op_add_reg(s, s->aflag, R_EDI, dshift);
1237	}
1238
1239	static void gen_cmps(DisasContext *s, MemOp ot, TCGv dshift)
1240	{
1241	s->cc_srcT = tcg_temp_new();
1242	gen_string_movl_A0_EDI(s);
1243	gen_op_ld_v(s, ot, s->T1, s->A0);
1244	gen_string_movl_A0_ESI(s);
1245	gen_op_ld_v(s, ot, s->T0, s->A0);
1246	tcg_gen_mov_tl(cpu_cc_src, s->T1);
1247	tcg_gen_mov_tl(s->cc_srcT, s->T0);
1248	tcg_gen_sub_tl(cpu_cc_dst, s->T0, s->T1);
1249	set_cc_op(s, CC_OP_SUBB + ot);
1250
1251	gen_op_add_reg(s, s->aflag, R_ESI, dshift);
1252	gen_op_add_reg(s, s->aflag, R_EDI, dshift);
1253	}
1254
1255	static void gen_bpt_io(DisasContext *s, TCGv_i32 t_port, int ot)
1256	{
1257	if (s->flags & HF_IOBPT_MASK) {
1258	#ifdef CONFIG_USER_ONLY
1259	/* user-mode cpu should not be in IOBPT mode */
1260	g_assert_not_reached();
1261	#else
1262	TCGv_i32 t_size = tcg_constant_i32(1 << ot);
1263	TCGv t_next = eip_next_tl(s);
1264	gen_helper_bpt_io(tcg_env, t_port, t_size, t_next);
1265	#endif /* CONFIG_USER_ONLY */
1266	}
1267	}
1268
1269	static void gen_ins(DisasContext *s, MemOp ot, TCGv dshift)
1270	{
1271	TCGv_i32 port = tcg_temp_new_i32();
1272
1273	gen_string_movl_A0_EDI(s);
1274	/* Note: we must do this dummy write first to be restartable in
1275	case of page fault. */
1276	tcg_gen_movi_tl(s->T0, 0);
1277	gen_op_st_v(s, ot, s->T0, s->A0);
1278	tcg_gen_trunc_tl_i32(port, cpu_regs[R_EDX]);
1279	tcg_gen_andi_i32(port, port, 0xffff);
1280	gen_helper_in_func(ot, s->T0, port);
1281	gen_op_st_v(s, ot, s->T0, s->A0);
1282	gen_op_add_reg(s, s->aflag, R_EDI, dshift);
1283	gen_bpt_io(s, port, ot);
1284	}
1285
1286	static void gen_outs(DisasContext *s, MemOp ot, TCGv dshift)
1287	{
1288	TCGv_i32 port = tcg_temp_new_i32();
1289	TCGv_i32 value = tcg_temp_new_i32();
1290
1291	gen_string_movl_A0_ESI(s);
1292	gen_op_ld_v(s, ot, s->T0, s->A0);
1293
1294	tcg_gen_trunc_tl_i32(port, cpu_regs[R_EDX]);
1295	tcg_gen_andi_i32(port, port, 0xffff);
1296	tcg_gen_trunc_tl_i32(value, s->T0);
1297	gen_helper_out_func(ot, port, value);
1298	gen_op_add_reg(s, s->aflag, R_ESI, dshift);
1299	gen_bpt_io(s, port, ot);
1300	}
1301
1302	#define REP_MAX 65535
1303
1304	static void do_gen_rep(DisasContext *s, MemOp ot, TCGv dshift,
1305	void (fn)(DisasContext s, MemOp ot, TCGv dshift),
1306	bool is_repz_nz)
1307	{
1308	TCGLabel *last = gen_new_label();
1309	TCGLabel *loop = gen_new_label();
1310	TCGLabel *done = gen_new_label();
1311
1312	target_ulong cx_mask = MAKE_64BIT_MASK(0, 8 << s->aflag);
1313	TCGv cx_next = tcg_temp_new();
1314
1315	/*
1316	* Check if we must translate a single iteration only. Normally, HF_RF_MASK
1317	* would also limit translation blocks to one instruction, so that gen_eob
1318	* can reset the flag; here however RF is set throughout the repetition, so
1319	* we can plow through until CX/ECX/RCX is zero.
1320	*/
1321	bool can_loop =
1322	(!(tb_cflags(s->base.tb) & (CF_USE_ICOUNT \| CF_SINGLE_STEP))
1323	&& !(s->flags & (HF_TF_MASK \| HF_INHIBIT_IRQ_MASK)));
1324	bool had_rf = s->flags & HF_RF_MASK;
1325
1326	/*
1327	* Even if EFLAGS.RF was set on entry (such as if we're on the second or
1328	* later iteration and an exception or interrupt happened), force gen_eob()
1329	* not to clear the flag. We do that ourselves after the last iteration.
1330	*/
1331	s->flags &= ~HF_RF_MASK;
1332
1333	/*
1334	* For CMPS/SCAS, the CC_OP after a memory fault could come from either
1335	* the previous instruction or the string instruction; but because we
1336	* arrange to keep CC_OP up to date all the time, just mark the whole
1337	* insn as CC_OP_DYNAMIC.
1338	*
1339	* It's not a problem to do this even for instructions that do not
1340	* modify the flags, so do it unconditionally.
1341	*/
1342	gen_update_cc_op(s);
1343	tcg_set_insn_start_param(s->base.insn_start, 1, CC_OP_DYNAMIC);
1344
1345	/* Any iteration at all? */
1346	tcg_gen_brcondi_tl(TCG_COND_TSTEQ, cpu_regs[R_ECX], cx_mask, done);
1347
1348	/*
1349	* From now on we operate on the value of CX/ECX/RCX that will be written
1350	* back, which is stored in cx_next. There can be no carry, so we can zero
1351	* extend here if needed and not do any expensive deposit operations later.
1352	*/
1353	tcg_gen_subi_tl(cx_next, cpu_regs[R_ECX], 1);
1354	#ifdef TARGET_X86_64
1355	if (s->aflag == MO_32) {
1356	tcg_gen_ext32u_tl(cx_next, cx_next);
1357	cx_mask = ~0;
1358	}
1359	#endif
1360
1361	/*
1362	* The last iteration is handled outside the loop, so that cx_next
1363	* can never underflow.
1364	*/
1365	if (can_loop) {
1366	tcg_gen_brcondi_tl(TCG_COND_TSTEQ, cx_next, cx_mask, last);
1367	}
1368
1369	gen_set_label(loop);
1370	fn(s, ot, dshift);
1371	tcg_gen_mov_tl(cpu_regs[R_ECX], cx_next);
1372	gen_update_cc_op(s);
1373
1374	/* Leave if REP condition fails. */
1375	if (is_repz_nz) {
1376	int nz = (s->prefix & PREFIX_REPNZ) ? 1 : 0;
1377	gen_jcc_noeob(s, (JCC_Z << 1) \| (nz ^ 1), done);
1378	/* gen_prepare_eflags_z never changes cc_op. */
1379	assert(!s->cc_op_dirty);
1380	}
1381
1382	if (can_loop) {
1383	tcg_gen_subi_tl(cx_next, cx_next, 1);
1384	tcg_gen_brcondi_tl(TCG_COND_TSTNE, cx_next, REP_MAX, loop);
1385	tcg_gen_brcondi_tl(TCG_COND_TSTEQ, cx_next, cx_mask, last);
1386	}
1387
1388	/*
1389	* Traps or interrupts set RF_MASK if they happen after any iteration
1390	* but the last. Set it here before giving the main loop a chance to
1391	* execute. (For faults, seg_helper.c sets the flag as usual).
1392	*/
1393	if (!had_rf) {
1394	gen_set_eflags(s, RF_MASK);
1395	}
1396
1397	/* Go to the main loop but reenter the same instruction. */
1398	gen_jmp_rel_csize(s, -cur_insn_len(s), 0);
1399
1400	if (can_loop) {
1401	/*
1402	* The last iteration needs no conditional jump, even if is_repz_nz,
1403	* because the repeats are ending anyway.
1404	*/
1405	gen_set_label(last);
1406	set_cc_op(s, CC_OP_DYNAMIC);
1407	fn(s, ot, dshift);
1408	tcg_gen_mov_tl(cpu_regs[R_ECX], cx_next);
1409	gen_update_cc_op(s);
1410	}
1411
1412	/* CX/ECX/RCX is zero, or REPZ/REPNZ broke the repetition. */
1413	gen_set_label(done);
1414	set_cc_op(s, CC_OP_DYNAMIC);
1415	if (had_rf) {
1416	gen_reset_eflags(s, RF_MASK);
1417	}
1418	gen_jmp_rel_csize(s, 0, 1);
1419	}
1420
1421	static void do_gen_string(DisasContext *s, MemOp ot,
1422	void (fn)(DisasContext s, MemOp ot, TCGv dshift),
1423	bool is_repz_nz)
1424	{
1425	TCGv dshift = tcg_temp_new();
1426	tcg_gen_ld32s_tl(dshift, tcg_env, offsetof(CPUX86State, df));
1427	tcg_gen_shli_tl(dshift, dshift, ot);
1428
1429	if (s->prefix & (PREFIX_REPZ \| PREFIX_REPNZ)) {
1430	do_gen_rep(s, ot, dshift, fn, is_repz_nz);
1431	} else {
1432	fn(s, ot, dshift);
1433	}
1434	}
1435
1436	static void gen_repz(DisasContext *s, MemOp ot,
1437	void (fn)(DisasContext s, MemOp ot, TCGv dshift))
1438	{
1439	do_gen_string(s, ot, fn, false);
1440	}
1441
1442	static void gen_repz_nz(DisasContext *s, MemOp ot,
1443	void (fn)(DisasContext s, MemOp ot, TCGv dshift))
1444	{
1445	do_gen_string(s, ot, fn, true);
1446	}
1447
1448	static void gen_helper_fp_arith_ST0_FT0(int op)
1449	{
1450	switch (op) {
1451	case 0:
1452	gen_helper_fadd_ST0_FT0(tcg_env);
1453	break;
1454	case 1:
1455	gen_helper_fmul_ST0_FT0(tcg_env);
1456	break;
1457	case 2:
1458	gen_helper_fcom_ST0_FT0(tcg_env);
1459	break;
1460	case 3:
1461	gen_helper_fcom_ST0_FT0(tcg_env);
1462	gen_helper_fpop(tcg_env);
1463	break;
1464	case 4:
1465	gen_helper_fsub_ST0_FT0(tcg_env);
1466	break;
1467	case 5:
1468	gen_helper_fsubr_ST0_FT0(tcg_env);
1469	break;
1470	case 6:
1471	gen_helper_fdiv_ST0_FT0(tcg_env);
1472	break;
1473	case 7:
1474	gen_helper_fdivr_ST0_FT0(tcg_env);
1475	break;
1476	}
1477	}
1478
1479	/* NOTE the exception in "r" op ordering */
1480	static void gen_helper_fp_arith_STN_ST0(int op, int opreg)
1481	{
1482	TCGv_i32 tmp = tcg_constant_i32(opreg);
1483	switch (op) {
1484	case 0:
1485	gen_helper_fadd_STN_ST0(tcg_env, tmp);
1486	break;
1487	case 1:
1488	gen_helper_fmul_STN_ST0(tcg_env, tmp);
1489	break;
1490	case 4:
1491	gen_helper_fsubr_STN_ST0(tcg_env, tmp);
1492	break;
1493	case 5:
1494	gen_helper_fsub_STN_ST0(tcg_env, tmp);
1495	break;
1496	case 6:
1497	gen_helper_fdivr_STN_ST0(tcg_env, tmp);
1498	break;
1499	case 7:
1500	gen_helper_fdiv_STN_ST0(tcg_env, tmp);
1501	break;
1502	}
1503	}
1504
1505	static void gen_exception(DisasContext *s, int trapno)
1506	{
1507	gen_update_cc_op(s);
1508	gen_update_eip_cur(s);
1509	gen_helper_raise_exception(tcg_env, tcg_constant_i32(trapno));
1510	s->base.is_jmp = DISAS_NORETURN;
1511	}
1512
1513	/* Generate #UD for the current instruction. The assumption here is that
1514	the instruction is known, but it isn't allowed in the current cpu mode. */
1515	static void gen_illegal_opcode(DisasContext *s)
1516	{
1517	gen_exception(s, EXCP06_ILLOP);
1518	}
1519
1520	/* Generate #GP for the current instruction. */
1521	static void gen_exception_gpf(DisasContext *s)
1522	{
1523	gen_exception(s, EXCP0D_GPF);
1524	}
1525
1526	/* Check for cpl == 0; if not, raise #GP and return false. */
1527	static bool check_cpl0(DisasContext *s)
1528	{
1529	if (CPL(s) == 0) {
1530	return true;
1531	}
1532	gen_exception_gpf(s);
1533	return false;
1534	}
1535
1536	/* XXX: add faster immediate case */
1537	static TCGv gen_shiftd_rm_T1(DisasContext *s, MemOp ot,
1538	bool is_right, TCGv count)
1539	{
1540	target_ulong mask = (ot == MO_64 ? 63 : 31);
1541	TCGv cc_src = tcg_temp_new();
1542	TCGv tmp = tcg_temp_new();
1543	TCGv hishift;
1544
1545	switch (ot) {
1546	case MO_16:
1547	/* Note: we implement the Intel behaviour for shift count > 16.
1548	This means "shrdw C, B, A" shifts A:B:A >> C. Build the B:A
1549	portion by constructing it as a 32-bit value. */
1550	if (is_right) {
1551	tcg_gen_deposit_tl(tmp, s->T0, s->T1, 16, 16);
1552	tcg_gen_mov_tl(s->T1, s->T0);
1553	tcg_gen_mov_tl(s->T0, tmp);
1554	} else {
1555	tcg_gen_deposit_tl(s->T1, s->T0, s->T1, 16, 16);
1556	}
1557	/*
1558	* If TARGET_X86_64 defined then fall through into MO_32 case,
1559	* otherwise fall through default case.
1560	*/
1561	case MO_32:
1562	#ifdef TARGET_X86_64
1563	/* Concatenate the two 32-bit values and use a 64-bit shift. */
1564	tcg_gen_subi_tl(tmp, count, 1);
1565	if (is_right) {
1566	tcg_gen_concat_tl_i64(s->T0, s->T0, s->T1);
1567	tcg_gen_shr_i64(cc_src, s->T0, tmp);
1568	tcg_gen_shr_i64(s->T0, s->T0, count);
1569	} else {
1570	tcg_gen_concat_tl_i64(s->T0, s->T1, s->T0);
1571	tcg_gen_shl_i64(cc_src, s->T0, tmp);
1572	tcg_gen_shl_i64(s->T0, s->T0, count);
1573	tcg_gen_shri_i64(cc_src, cc_src, 32);
1574	tcg_gen_shri_i64(s->T0, s->T0, 32);
1575	}
1576	break;
1577	case MO_64:
1578	#endif
1579	hishift = tcg_temp_new();
1580	tcg_gen_subi_tl(tmp, count, 1);
1581	if (is_right) {
1582	tcg_gen_shr_tl(cc_src, s->T0, tmp);
1583
1584	/* mask + 1 - count = mask - tmp = mask ^ tmp */
1585	tcg_gen_xori_tl(hishift, tmp, mask);
1586	tcg_gen_shr_tl(s->T0, s->T0, count);
1587	tcg_gen_shl_tl(s->T1, s->T1, hishift);
1588	} else {
1589	tcg_gen_shl_tl(cc_src, s->T0, tmp);
1590
1591	/* mask + 1 - count = mask - tmp = mask ^ tmp */
1592	tcg_gen_xori_tl(hishift, tmp, mask);
1593	tcg_gen_shl_tl(s->T0, s->T0, count);
1594	tcg_gen_shr_tl(s->T1, s->T1, hishift);
1595
1596	if (ot == MO_16) {
1597	/* Only needed if count > 16, for Intel behaviour. */
1598	tcg_gen_shri_tl(tmp, s->T1, 1);
1599	tcg_gen_or_tl(cc_src, cc_src, tmp);
1600	}
1601	}
1602	tcg_gen_movcond_tl(TCG_COND_EQ, s->T1,
1603	count, tcg_constant_tl(0),
1604	tcg_constant_tl(0), s->T1);
1605	tcg_gen_or_tl(s->T0, s->T0, s->T1);
1606	break;
1607
1608	default:
1609	g_assert_not_reached();
1610	}
1611
1612	return cc_src;
1613	}
1614
1615	/* Compute the address, with a minimum number of TCG ops. */
1616	static TCGv gen_lea_modrm_1(DisasContext *s, AddressParts a, bool is_vsib)
1617	{
1618	TCGv ea = NULL;
1619
1620	if (a.index >= 0 && !is_vsib) {
1621	if (a.scale == 0) {
1622	ea = cpu_regs[a.index];
1623	} else {
1624	tcg_gen_shli_tl(s->A0, cpu_regs[a.index], a.scale);
1625	ea = s->A0;
1626	}
1627	if (a.base >= 0) {
1628	tcg_gen_add_tl(s->A0, ea, cpu_regs[a.base]);
1629	ea = s->A0;
1630	}
1631	} else if (a.base >= 0) {
1632	ea = cpu_regs[a.base];
1633	}
1634	if (!ea) {
1635	if (tb_cflags(s->base.tb) & CF_PCREL && a.base == -2) {
1636	/* With cpu_eip ~= pc_save, the expression is pc-relative. */
1637	tcg_gen_addi_tl(s->A0, cpu_eip, a.disp - s->pc_save);
1638	} else {
1639	tcg_gen_movi_tl(s->A0, a.disp);
1640	}
1641	ea = s->A0;
1642	} else if (a.disp != 0) {
1643	tcg_gen_addi_tl(s->A0, ea, a.disp);
1644	ea = s->A0;
1645	}
1646
1647	return ea;
1648	}
1649
1650	/* Used for BNDCL, BNDCU, BNDCN. */
1651	static void gen_bndck(DisasContext s, X86DecodedInsn decode,
1652	TCGCond cond, TCGv_i64 bndv)
1653	{
1654	TCGv ea = gen_lea_modrm_1(s, decode->mem, false);
1655	TCGv_i32 t32 = tcg_temp_new_i32();
1656	TCGv_i64 t64 = tcg_temp_new_i64();
1657
1658	tcg_gen_extu_tl_i64(t64, ea);
1659	if (!CODE64(s)) {
1660	tcg_gen_ext32u_i64(t64, t64);
1661	}
1662	tcg_gen_setcond_i64(cond, t64, t64, bndv);
1663	tcg_gen_extrl_i64_i32(t32, t64);
1664	gen_helper_bndck(tcg_env, t32);
1665	}
1666
1667	/* generate modrm load of memory or register. */
1668	static void gen_ld_modrm(DisasContext s, X86DecodedInsn decode, MemOp ot)
1669	{
1670	int modrm = s->modrm;
1671	int mod, rm;
1672
1673	mod = (modrm >> 6) & 3;
1674	rm = (modrm & 7) \| REX_B(s);
1675	if (mod == 3) {
1676	gen_op_mov_v_reg(s, ot, s->T0, rm);
1677	} else {
1678	gen_lea_modrm(s, decode);
1679	gen_op_ld_v(s, ot, s->T0, s->A0);
1680	}
1681	}
1682
1683	/* generate modrm store of memory or register. */
1684	static void gen_st_modrm(DisasContext s, X86DecodedInsn decode, MemOp ot)
1685	{
1686	int modrm = s->modrm;
1687	int mod, rm;
1688
1689	mod = (modrm >> 6) & 3;
1690	rm = (modrm & 7) \| REX_B(s);
1691	if (mod == 3) {
1692	gen_op_mov_reg_v(s, ot, rm, s->T0);
1693	} else {
1694	gen_lea_modrm(s, decode);
1695	gen_op_st_v(s, ot, s->T0, s->A0);
1696	}
1697	}
1698
1699	static void gen_conditional_jump_labels(DisasContext *s, target_long diff,
1700	TCGLabel not_taken, TCGLabel taken)
1701	{
1702	if (not_taken) {
1703	gen_set_label(not_taken);
1704	}
1705	gen_jmp_rel_csize(s, 0, 1);
1706
1707	gen_set_label(taken);
1708	gen_jmp_rel(s, s->dflag, diff, 0);
1709	}
1710
1711	static void gen_cmovcc(DisasContext *s, int b, TCGv dest, TCGv src)
1712	{
1713	CCPrepare cc = gen_prepare_cc(s, b, NULL);
1714
1715	tcg_gen_movcond_tl(cc.cond, dest, cc.reg, cc.reg2, src, dest);
1716	}
1717
1718	static void gen_op_movl_seg_real(DisasContext *s, X86Seg seg_reg, TCGv seg)
1719	{
1720	TCGv selector = tcg_temp_new();
1721	tcg_gen_ext16u_tl(selector, seg);
1722	tcg_gen_st32_tl(selector, tcg_env,
1723	offsetof(CPUX86State,segs[seg_reg].selector));
1724	tcg_gen_shli_tl(cpu_seg_base[seg_reg], selector, 4);
1725	}
1726
1727	/* move SRC to seg_reg and compute if the CPU state may change. Never
1728	call this function with seg_reg == R_CS */
1729	static void gen_movl_seg(DisasContext *s, X86Seg seg_reg, TCGv src, bool inhibit_irq)
1730	{
1731	if (PE(s) && !VM86(s)) {
1732	TCGv_i32 sel = tcg_temp_new_i32();
1733
1734	tcg_gen_trunc_tl_i32(sel, src);
1735	gen_helper_load_seg(tcg_env, tcg_constant_i32(seg_reg), sel);
1736
1737	/*
1738	* For moves to SS, the SS32 flag may change. For CODE32 only, changes
1739	* to SS, DS and ES may change the ADDSEG flags.
1740	*/
1741	if (seg_reg == R_SS \|\| (CODE32(s) && seg_reg < R_FS)) {
1742	s->base.is_jmp = DISAS_EOB_NEXT;
1743	}
1744	} else {
1745	gen_op_movl_seg_real(s, seg_reg, src);
1746	}
1747
1748	/*
1749	* For MOV or POP to SS (but not LSS) translation must always
1750	* stop as a special handling must be done to disable hardware
1751	* interrupts for the next instruction.
1752	*
1753	* This is the last instruction, so it's okay to overwrite
1754	* HF_TF_MASK; the next TB will start with the flag set.
1755	*
1756	* DISAS_EOB_INHIBIT_IRQ is a superset of DISAS_EOB_NEXT which
1757	* might have been set above.
1758	*/
1759	if (inhibit_irq) {
1760	s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ;
1761	s->flags &= ~HF_TF_MASK;
1762	}
1763	}
1764
1765	static void gen_far_call(DisasContext *s)
1766	{
1767	TCGv_i32 new_cs = tcg_temp_new_i32();
1768	tcg_gen_trunc_tl_i32(new_cs, s->T1);
1769	if (PE(s) && !VM86(s)) {
1770	gen_helper_lcall_protected(tcg_env, new_cs, s->T0,
1771	tcg_constant_i32(s->dflag - 1),
1772	eip_next_tl(s));
1773	} else {
1774	TCGv_i32 new_eip = tcg_temp_new_i32();
1775	tcg_gen_trunc_tl_i32(new_eip, s->T0);
1776	gen_helper_lcall_real(tcg_env, new_cs, new_eip,
1777	tcg_constant_i32(s->dflag - 1),
1778	eip_next_i32(s));
1779	}
1780	s->base.is_jmp = DISAS_JUMP;
1781	}
1782
1783	static void gen_far_jmp(DisasContext *s)
1784	{
1785	if (PE(s) && !VM86(s)) {
1786	TCGv_i32 new_cs = tcg_temp_new_i32();
1787	tcg_gen_trunc_tl_i32(new_cs, s->T1);
1788	gen_helper_ljmp_protected(tcg_env, new_cs, s->T0,
1789	eip_next_tl(s));
1790	} else {
1791	gen_op_movl_seg_real(s, R_CS, s->T1);
1792	gen_op_jmp_v(s, s->T0);
1793	}
1794	s->base.is_jmp = DISAS_JUMP;
1795	}
1796
1797	static void gen_svm_check_intercept(DisasContext *s, uint32_t type)
1798	{
1799	/* no SVM activated; fast case */
1800	if (likely(!GUEST(s))) {
1801	return;
1802	}
1803	gen_helper_svm_check_intercept(tcg_env, tcg_constant_i32(type));
1804	}
1805
1806	static inline void gen_stack_update(DisasContext *s, int addend)
1807	{
1808	gen_op_add_reg_im(s, mo_stacksize(s), R_ESP, addend);
1809	}
1810
1811	static void gen_lea_ss_ofs(DisasContext *s, TCGv dest, TCGv src, target_ulong offset)
1812	{
1813	if (offset) {
1814	tcg_gen_addi_tl(dest, src, offset);
1815	src = dest;
1816	}
1817	gen_lea_v_seg_dest(s, mo_stacksize(s), dest, src, R_SS, -1);
1818	}
1819
1820	/* Generate a push. It depends on ss32, addseg and dflag. */
1821	static void gen_push_v(DisasContext *s, TCGv val)
1822	{
1823	MemOp d_ot = mo_pushpop(s, s->dflag);
1824	MemOp a_ot = mo_stacksize(s);
1825	int size = 1 << d_ot;
1826	TCGv new_esp = tcg_temp_new();
1827
1828	tcg_gen_subi_tl(new_esp, cpu_regs[R_ESP], size);
1829
1830	/* Now reduce the value to the address size and apply SS base. */
1831	gen_lea_ss_ofs(s, s->A0, new_esp, 0);
1832	gen_op_st_v(s, d_ot, val, s->A0);
1833	gen_op_mov_reg_v(s, a_ot, R_ESP, new_esp);
1834	}
1835
1836	/* two step pop is necessary for precise exceptions */
1837	static MemOp gen_pop_T0(DisasContext *s)
1838	{
1839	MemOp d_ot = mo_pushpop(s, s->dflag);
1840
1841	gen_lea_ss_ofs(s, s->T0, cpu_regs[R_ESP], 0);
1842	gen_op_ld_v(s, d_ot, s->T0, s->T0);
1843
1844	return d_ot;
1845	}
1846
1847	static inline void gen_pop_update(DisasContext *s, MemOp ot)
1848	{
1849	gen_stack_update(s, 1 << ot);
1850	}
1851
1852	static void gen_pusha(DisasContext *s)
1853	{
1854	MemOp d_ot = s->dflag;
1855	int size = 1 << d_ot;
1856	int i;
1857
1858	for (i = 0; i < 8; i++) {
1859	gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], (i - 8) * size);
1860	gen_op_st_v(s, d_ot, cpu_regs[7 - i], s->A0);
1861	}
1862
1863	gen_stack_update(s, -8 * size);
1864	}
1865
1866	static void gen_popa(DisasContext *s)
1867	{
1868	MemOp d_ot = s->dflag;
1869	int size = 1 << d_ot;
1870	int i;
1871
1872	for (i = 0; i < 8; i++) {
1873	/* ESP is not reloaded */
1874	if (7 - i == R_ESP) {
1875	continue;
1876	}
1877	gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], i * size);
1878	gen_op_ld_v(s, d_ot, s->T0, s->A0);
1879	gen_op_mov_reg_v(s, d_ot, 7 - i, s->T0);
1880	}
1881
1882	gen_stack_update(s, 8 * size);
1883	}
1884
1885	static void gen_enter(DisasContext *s, int esp_addend, int level)
1886	{
1887	MemOp d_ot = mo_pushpop(s, s->dflag);
1888	MemOp a_ot = mo_stacksize(s);
1889	int size = 1 << d_ot;
1890
1891	/* Push BP; compute FrameTemp into T1. */
1892	tcg_gen_subi_tl(s->T1, cpu_regs[R_ESP], size);
1893	gen_lea_ss_ofs(s, s->A0, s->T1, 0);
1894	gen_op_st_v(s, d_ot, cpu_regs[R_EBP], s->A0);
1895
1896	level &= 31;
1897	if (level != 0) {
1898	int i;
1899	if (level > 1) {
1900	TCGv fp = tcg_temp_new();
1901
1902	/* Copy level-1 pointers from the previous frame. */
1903	for (i = 1; i < level; ++i) {
1904	gen_lea_ss_ofs(s, s->A0, cpu_regs[R_EBP], -size * i);
1905	gen_op_ld_v(s, d_ot, fp, s->A0);
1906
1907	gen_lea_ss_ofs(s, s->A0, s->T1, -size * i);
1908	gen_op_st_v(s, d_ot, fp, s->A0);
1909	}
1910	}
1911
1912	/* Push the current FrameTemp as the last level. */
1913	gen_lea_ss_ofs(s, s->A0, s->T1, -size * level);
1914	gen_op_st_v(s, d_ot, s->T1, s->A0);
1915	}
1916
1917	/* Copy the FrameTemp value to EBP. */
1918	gen_op_mov_reg_v(s, d_ot, R_EBP, s->T1);
1919
1920	/* Compute the final value of ESP. */
1921	tcg_gen_subi_tl(s->T1, s->T1, esp_addend + size * level);
1922	gen_op_mov_reg_v(s, a_ot, R_ESP, s->T1);
1923	}
1924
1925	static void gen_leave(DisasContext *s)
1926	{
1927	MemOp d_ot = mo_pushpop(s, s->dflag);
1928	MemOp a_ot = mo_stacksize(s);
1929
1930	gen_lea_ss_ofs(s, s->A0, cpu_regs[R_EBP], 0);
1931	gen_op_ld_v(s, d_ot, s->T0, s->A0);
1932
1933	tcg_gen_addi_tl(s->T1, cpu_regs[R_EBP], 1 << d_ot);
1934
1935	gen_op_mov_reg_v(s, d_ot, R_EBP, s->T0);
1936	gen_op_mov_reg_v(s, a_ot, R_ESP, s->T1);
1937	}
1938
1939	/* an interrupt is different from an exception because of the
1940	privilege checks */
1941	static void gen_interrupt(DisasContext *s, uint8_t intno)
1942	{
1943	gen_update_cc_op(s);
1944	gen_update_eip_cur(s);
1945	gen_helper_raise_interrupt(tcg_env, tcg_constant_i32(intno),
1946	cur_insn_len_i32(s));
1947	s->base.is_jmp = DISAS_NORETURN;
1948	}
1949
1950	/* Clear BND registers during legacy branches. */
1951	static void gen_bnd_jmp(DisasContext *s)
1952	{
1953	/* Clear the registers only if BND prefix is missing, MPX is enabled,
1954	and if the BNDREGs are known to be in use (non-zero) already.
1955	The helper itself will check BNDPRESERVE at runtime. */
1956	if ((s->prefix & PREFIX_REPNZ) == 0
1957	&& (s->flags & HF_MPX_EN_MASK) != 0
1958	&& (s->flags & HF_MPX_IU_MASK) != 0) {
1959	gen_helper_bnd_jmp(tcg_env);
1960	}
1961	}
1962
1963	/*
1964	* Generate an end of block, including common tasks such as generating
1965	* single step traps, resetting the RF flag, and handling the interrupt
1966	* shadow.
1967	*/
1968	static void
1969	gen_eob(DisasContext *s, int mode)
1970	{
1971	bool inhibit_reset;
1972
1973	gen_update_cc_op(s);
1974
1975	/* If several instructions disable interrupts, only the first does it. */
1976	inhibit_reset = false;
1977	if (s->flags & HF_INHIBIT_IRQ_MASK) {
1978	gen_reset_hflag(s, HF_INHIBIT_IRQ_MASK);
1979	inhibit_reset = true;
1980	} else if (mode == DISAS_EOB_INHIBIT_IRQ) {
1981	gen_set_hflag(s, HF_INHIBIT_IRQ_MASK);
1982	}
1983
1984	if (s->flags & HF_RF_MASK) {
1985	gen_reset_eflags(s, RF_MASK);
1986	}
1987	if (mode == DISAS_EOB_RECHECK_TF) {
1988	gen_helper_rechecking_single_step(tcg_env);
1989	tcg_gen_exit_tb(NULL, 0);
1990	} else if (s->flags & HF_TF_MASK) {
1991	gen_helper_single_step(tcg_env);
1992	} else if (mode == DISAS_JUMP &&
1993	/* give irqs a chance to happen */
1994	!inhibit_reset) {
1995	tcg_gen_lookup_and_goto_ptr();
1996	} else {
1997	tcg_gen_exit_tb(NULL, 0);
1998	}
1999
2000	s->base.is_jmp = DISAS_NORETURN;
2001	}
2002
2003	/* Jump to eip+diff, truncating the result to OT. */
2004	static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num)
2005	{
2006	bool use_goto_tb = s->jmp_opt;
2007	target_ulong mask = -1;
2008	target_ulong new_pc = s->pc + diff;
2009	target_ulong new_eip = new_pc - s->cs_base;
2010
2011	assert(!s->cc_op_dirty);
2012
2013	/* In 64-bit mode, operand size is fixed at 64 bits. */
2014	if (!CODE64(s)) {
2015	if (ot == MO_16) {
2016	mask = 0xffff;
2017	if (tb_cflags(s->base.tb) & CF_PCREL && CODE32(s)) {
2018	use_goto_tb = false;
2019	}
2020	} else {
2021	mask = 0xffffffff;
2022	}
2023	}
2024	new_eip &= mask;
2025
2026	if (tb_cflags(s->base.tb) & CF_PCREL) {
2027	tcg_gen_addi_tl(cpu_eip, cpu_eip, new_pc - s->pc_save);
2028	/*
2029	* If we can prove the branch does not leave the page and we have
2030	* no extra masking to apply (data16 branch in code32, see above),
2031	* then we have also proven that the addition does not wrap.
2032	*/
2033	if (!use_goto_tb \|\| !translator_is_same_page(&s->base, new_pc)) {
2034	tcg_gen_andi_tl(cpu_eip, cpu_eip, mask);
2035	use_goto_tb = false;
2036	}
2037	} else if (!CODE64(s)) {
2038	new_pc = (uint32_t)(new_eip + s->cs_base);
2039	}
2040
2041	if (use_goto_tb && translator_use_goto_tb(&s->base, new_pc)) {
2042	/* jump to same page: we can use a direct jump */
2043	tcg_gen_goto_tb(tb_num);
2044	if (!(tb_cflags(s->base.tb) & CF_PCREL)) {
2045	tcg_gen_movi_tl(cpu_eip, new_eip);
2046	}
2047	tcg_gen_exit_tb(s->base.tb, tb_num);
2048	s->base.is_jmp = DISAS_NORETURN;
2049	} else {
2050	if (!(tb_cflags(s->base.tb) & CF_PCREL)) {
2051	tcg_gen_movi_tl(cpu_eip, new_eip);
2052	}
2053	if (s->jmp_opt) {
2054	gen_eob(s, DISAS_JUMP); /* jump to another page */
2055	} else {
2056	gen_eob(s, DISAS_EOB_ONLY); /* exit to main loop */
2057	}
2058	}
2059	}
2060
2061	/* Jump to eip+diff, truncating to the current code size. */
2062	static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num)
2063	{
2064	/* CODE64 ignores the OT argument, so we need not consider it. */
2065	gen_jmp_rel(s, CODE32(s) ? MO_32 : MO_16, diff, tb_num);
2066	}
2067
2068	static inline void gen_ldq_env_A0(DisasContext *s, int offset)
2069	{
2070	TCGv_i64 t = tcg_temp_new_i64();
2071
2072	tcg_gen_qemu_ld_i64(t, s->A0, s->mem_index, MO_LEUQ);
2073	tcg_gen_st_i64(t, tcg_env, offset);
2074	}
2075
2076	static inline void gen_stq_env_A0(DisasContext *s, int offset)
2077	{
2078	TCGv_i64 t = tcg_temp_new_i64();
2079
2080	tcg_gen_ld_i64(t, tcg_env, offset);
2081	tcg_gen_qemu_st_i64(t, s->A0, s->mem_index, MO_LEUQ);
2082	}
2083
2084	static inline void gen_ldo_env_A0(DisasContext *s, int offset, bool align)
2085	{
2086	MemOp atom = (s->cpuid_ext_features & CPUID_EXT_AVX
2087	? MO_ATOM_IFALIGN : MO_ATOM_IFALIGN_PAIR);
2088	MemOp mop = MO_128 \| MO_LE \| atom \| (align ? MO_ALIGN_16 : 0);
2089	int mem_index = s->mem_index;
2090	TCGv_i128 t = tcg_temp_new_i128();
2091
2092	tcg_gen_qemu_ld_i128(t, s->A0, mem_index, mop);
2093	tcg_gen_st_i128(t, tcg_env, offset);
2094	}
2095
2096	static inline void gen_sto_env_A0(DisasContext *s, int offset, bool align)
2097	{
2098	MemOp atom = (s->cpuid_ext_features & CPUID_EXT_AVX
2099	? MO_ATOM_IFALIGN : MO_ATOM_IFALIGN_PAIR);
2100	MemOp mop = MO_128 \| MO_LE \| atom \| (align ? MO_ALIGN_16 : 0);
2101	int mem_index = s->mem_index;
2102	TCGv_i128 t = tcg_temp_new_i128();
2103
2104	tcg_gen_ld_i128(t, tcg_env, offset);
2105	tcg_gen_qemu_st_i128(t, s->A0, mem_index, mop);
2106	}
2107
2108	static void gen_ldy_env_A0(DisasContext *s, int offset, bool align)
2109	{
2110	MemOp mop = MO_128 \| MO_LE \| MO_ATOM_IFALIGN_PAIR;
2111	int mem_index = s->mem_index;
2112	TCGv_i128 t0 = tcg_temp_new_i128();
2113	TCGv_i128 t1 = tcg_temp_new_i128();
2114	TCGv a0_hi = tcg_temp_new();
2115
2116	tcg_gen_qemu_ld_i128(t0, s->A0, mem_index, mop \| (align ? MO_ALIGN_32 : 0));
2117	tcg_gen_addi_tl(a0_hi, s->A0, 16);
2118	tcg_gen_qemu_ld_i128(t1, a0_hi, mem_index, mop);
2119
2120	tcg_gen_st_i128(t0, tcg_env, offset + offsetof(YMMReg, YMM_X(0)));
2121	tcg_gen_st_i128(t1, tcg_env, offset + offsetof(YMMReg, YMM_X(1)));
2122	}
2123
2124	static void gen_sty_env_A0(DisasContext *s, int offset, bool align)
2125	{
2126	MemOp mop = MO_128 \| MO_LE \| MO_ATOM_IFALIGN_PAIR;
2127	int mem_index = s->mem_index;
2128	TCGv_i128 t = tcg_temp_new_i128();
2129	TCGv a0_hi = tcg_temp_new();
2130
2131	tcg_gen_ld_i128(t, tcg_env, offset + offsetof(YMMReg, YMM_X(0)));
2132	tcg_gen_qemu_st_i128(t, s->A0, mem_index, mop \| (align ? MO_ALIGN_32 : 0));
2133	tcg_gen_addi_tl(a0_hi, s->A0, 16);
2134	tcg_gen_ld_i128(t, tcg_env, offset + offsetof(YMMReg, YMM_X(1)));
2135	tcg_gen_qemu_st_i128(t, a0_hi, mem_index, mop);
2136	}
2137
2138	#include "emit.c.inc"
2139
2140	static void gen_x87(DisasContext s, X86DecodedInsn decode)
2141	{
2142	bool update_fip = true;
2143	int b = decode->b;
2144	int modrm = s->modrm;
2145	int mod, rm, op;
2146
2147	if (s->flags & (HF_EM_MASK \| HF_TS_MASK)) {
2148	/* if CR0.EM or CR0.TS are set, generate an FPU exception */
2149	/* XXX: what to do if illegal op ? */
2150	gen_exception(s, EXCP07_PREX);
2151	return;
2152	}
2153	mod = (modrm >> 6) & 3;
2154	rm = modrm & 7;
2155	op = ((b & 7) << 3) \| ((modrm >> 3) & 7);
2156	if (mod != 3) {
2157	/* memory op */
2158	TCGv ea = gen_lea_modrm_1(s, decode->mem, false);
2159	TCGv last_addr = tcg_temp_new();
2160	bool update_fdp = true;
2161	TCGv_i32 t32;
2162	TCGv_i64 t64;
2163
2164	tcg_gen_mov_tl(last_addr, ea);
2165	gen_lea_v_seg(s, ea, decode->mem.def_seg, s->override);
2166
2167	switch (op) {
2168	case 0x00 ... 0x07: /* fxxxs */
2169	t32 = tcg_temp_new_i32();
2170	tcg_gen_qemu_ld_i32(t32, s->A0,
2171	s->mem_index, MO_LEUL);
2172	gen_helper_flds_FT0(tcg_env, t32);
2173	gen_helper_fp_arith_ST0_FT0(op & 7);
2174	break;
2175
2176	case 0x10 ... 0x17: /* fixxxl */
2177	t32 = tcg_temp_new_i32();
2178	tcg_gen_qemu_ld_i32(t32, s->A0,
2179	s->mem_index, MO_LEUL);
2180	gen_helper_fildl_FT0(tcg_env, t32);
2181	gen_helper_fp_arith_ST0_FT0(op & 7);
2182	break;
2183
2184	case 0x20 ... 0x27: /* fxxxl */
2185	t64 = tcg_temp_new_i64();
2186	tcg_gen_qemu_ld_i64(t64, s->A0,
2187	s->mem_index, MO_LEUQ);
2188	gen_helper_fldl_FT0(tcg_env, t64);
2189	gen_helper_fp_arith_ST0_FT0(op & 7);
2190	break;
2191
2192	case 0x30 ... 0x37: /* fixxx */
2193	t32 = tcg_temp_new_i32();
2194	tcg_gen_qemu_ld_i32(t32, s->A0,
2195	s->mem_index, MO_LESW);
2196	gen_helper_fildl_FT0(tcg_env, t32);
2197	gen_helper_fp_arith_ST0_FT0(op & 7);
2198	break;
2199
2200	case 0x08: /* flds */
2201	t32 = tcg_temp_new_i32();
2202	tcg_gen_qemu_ld_i32(t32, s->A0,
2203	s->mem_index, MO_LEUL);
2204	gen_helper_flds_ST0(tcg_env, t32);
2205	break;
2206	case 0x18: /* fildl */
2207	t32 = tcg_temp_new_i32();
2208	tcg_gen_qemu_ld_i32(t32, s->A0,
2209	s->mem_index, MO_LEUL);
2210	gen_helper_fildl_ST0(tcg_env, t32);
2211	break;
2212	case 0x28: /* fldl */
2213	t64 = tcg_temp_new_i64();
2214	tcg_gen_qemu_ld_i64(t64, s->A0,
2215	s->mem_index, MO_LEUQ);
2216	gen_helper_fldl_ST0(tcg_env, t64);
2217	break;
2218	case 0x38: /* filds */
2219	t32 = tcg_temp_new_i32();
2220	tcg_gen_qemu_ld_i32(t32, s->A0,
2221	s->mem_index, MO_LESW);
2222	gen_helper_fildl_ST0(tcg_env, t32);
2223	break;
2224
2225	case 0x19: /* fisttpl */
2226	t32 = tcg_temp_new_i32();
2227	gen_helper_fisttl_ST0(t32, tcg_env);
2228	tcg_gen_qemu_st_i32(t32, s->A0,
2229	s->mem_index, MO_LEUL);
2230	gen_helper_fpop(tcg_env);
2231	break;
2232	case 0x29: /* fisttpll */
2233	t64 = tcg_temp_new_i64();
2234	gen_helper_fisttll_ST0(t64, tcg_env);
2235	tcg_gen_qemu_st_i64(t64, s->A0,
2236	s->mem_index, MO_LEUQ);
2237	gen_helper_fpop(tcg_env);
2238	break;
2239	case 0x39: /* fisttps */
2240	t32 = tcg_temp_new_i32();
2241	gen_helper_fistt_ST0(t32, tcg_env);
2242	tcg_gen_qemu_st_i32(t32, s->A0,
2243	s->mem_index, MO_LEUW);
2244	gen_helper_fpop(tcg_env);
2245	break;
2246
2247	case 0x0a: case 0x0b: /* fsts, fstps */
2248	t32 = tcg_temp_new_i32();
2249	gen_helper_fsts_ST0(t32, tcg_env);
2250	tcg_gen_qemu_st_i32(t32, s->A0,
2251	s->mem_index, MO_LEUL);
2252	if ((op & 7) == 3) {
2253	gen_helper_fpop(tcg_env);
2254	}
2255	break;
2256	case 0x1a: case 0x1b: /* fistl, fistpl */
2257	t32 = tcg_temp_new_i32();
2258	gen_helper_fistl_ST0(t32, tcg_env);
2259	tcg_gen_qemu_st_i32(t32, s->A0,
2260	s->mem_index, MO_LEUL);
2261	if ((op & 7) == 3) {
2262	gen_helper_fpop(tcg_env);
2263	}
2264	break;
2265	case 0x2a: case 0x2b: /* fstl, fstpl */
2266	t64 = tcg_temp_new_i64();
2267	gen_helper_fstl_ST0(t64, tcg_env);
2268	tcg_gen_qemu_st_i64(t64, s->A0,
2269	s->mem_index, MO_LEUQ);
2270	if ((op & 7) == 3) {
2271	gen_helper_fpop(tcg_env);
2272	}
2273	break;
2274
2275	case 0x3a: case 0x3b: /* fists, fistps */
2276	t32 = tcg_temp_new_i32();
2277	gen_helper_fist_ST0(t32, tcg_env);
2278	tcg_gen_qemu_st_i32(t32, s->A0,
2279	s->mem_index, MO_LEUW);
2280	if ((op & 7) == 3) {
2281	gen_helper_fpop(tcg_env);
2282	}
2283	break;
2284	case 0x0c: /* fldenv mem */
2285	gen_helper_fldenv(tcg_env, s->A0,
2286	tcg_constant_i32(s->dflag - 1));
2287	update_fip = update_fdp = false;
2288	break;
2289	case 0x0d: /* fldcw mem */
2290	t32 = tcg_temp_new_i32();
2291	tcg_gen_qemu_ld_i32(t32, s->A0,
2292	s->mem_index, MO_LEUW);
2293	gen_helper_fldcw(tcg_env, t32);
2294	update_fip = update_fdp = false;
2295	break;
2296	case 0x0e: /* fnstenv mem */
2297	gen_helper_fstenv(tcg_env, s->A0,
2298	tcg_constant_i32(s->dflag - 1));
2299	update_fip = update_fdp = false;
2300	break;
2301	case 0x0f: /* fnstcw mem */
2302	t32 = tcg_temp_new_i32();
2303	gen_helper_fnstcw(t32, tcg_env);
2304	tcg_gen_qemu_st_i32(t32, s->A0,
2305	s->mem_index, MO_LEUW);
2306	update_fip = update_fdp = false;
2307	break;
2308	case 0x1d: /* fldt mem */
2309	gen_helper_fldt_ST0(tcg_env, s->A0);
2310	break;
2311	case 0x1f: /* fstpt mem */
2312	gen_helper_fstt_ST0(tcg_env, s->A0);
2313	gen_helper_fpop(tcg_env);
2314	break;
2315	case 0x2c: /* frstor mem */
2316	gen_helper_frstor(tcg_env, s->A0,
2317	tcg_constant_i32(s->dflag - 1));
2318	update_fip = update_fdp = false;
2319	break;
2320	case 0x2e: /* fnsave mem */
2321	gen_helper_fsave(tcg_env, s->A0,
2322	tcg_constant_i32(s->dflag - 1));
2323	update_fip = update_fdp = false;
2324	break;
2325	case 0x2f: /* fnstsw mem */
2326	t32 = tcg_temp_new_i32();
2327	gen_helper_fnstsw(t32, tcg_env);
2328	tcg_gen_qemu_st_i32(t32, s->A0,
2329	s->mem_index, MO_LEUW);
2330	update_fip = update_fdp = false;
2331	break;
2332	case 0x3c: /* fbld */
2333	gen_helper_fbld_ST0(tcg_env, s->A0);
2334	break;
2335	case 0x3e: /* fbstp */
2336	gen_helper_fbst_ST0(tcg_env, s->A0);
2337	gen_helper_fpop(tcg_env);
2338	break;
2339	case 0x3d: /* fildll */
2340	t64 = tcg_temp_new_i64();
2341	tcg_gen_qemu_ld_i64(t64, s->A0,
2342	s->mem_index, MO_LEUQ);
2343	gen_helper_fildll_ST0(tcg_env, t64);
2344	break;
2345	case 0x3f: /* fistpll */
2346	t64 = tcg_temp_new_i64();
2347	gen_helper_fistll_ST0(t64, tcg_env);
2348	tcg_gen_qemu_st_i64(t64, s->A0,
2349	s->mem_index, MO_LEUQ);
2350	gen_helper_fpop(tcg_env);
2351	break;
2352	default:
2353	goto illegal_op;
2354	}
2355
2356	if (update_fdp) {
2357	int last_seg = s->override >= 0 ? s->override : decode->mem.def_seg;
2358
2359	t32 = tcg_temp_new_i32();
2360	tcg_gen_ld_i32(t32, tcg_env,
2361	offsetof(CPUX86State,
2362	segs[last_seg].selector));
2363	tcg_gen_st16_i32(t32, tcg_env,
2364	offsetof(CPUX86State, fpds));
2365	tcg_gen_st_tl(last_addr, tcg_env,
2366	offsetof(CPUX86State, fpdp));
2367	}
2368	} else {
2369	/* register float ops */
2370	int opreg = rm;
2371	TCGv_i32 t32;
2372
2373	switch (op) {
2374	case 0x08: /* fld sti */
2375	gen_helper_fpush(tcg_env);
2376	gen_helper_fmov_ST0_STN(tcg_env,
2377	tcg_constant_i32((opreg + 1) & 7));
2378	break;
2379	case 0x09: /* fxchg sti */
2380	case 0x29: /* fxchg4 sti, undocumented op */
2381	case 0x39: /* fxchg7 sti, undocumented op */
2382	gen_helper_fxchg_ST0_STN(tcg_env, tcg_constant_i32(opreg));
2383	break;
2384	case 0x0a: /* grp d9/2 */
2385	switch (rm) {
2386	case 0: /* fnop */
2387	/*
2388	* check exceptions (FreeBSD FPU probe)
2389	* needs to be treated as I/O because of ferr_irq
2390	*/
2391	translator_io_start(&s->base);
2392	gen_helper_fwait(tcg_env);
2393	update_fip = false;
2394	break;
2395	default:
2396	goto illegal_op;
2397	}
2398	break;
2399	case 0x0c: /* grp d9/4 */
2400	switch (rm) {
2401	case 0: /* fchs */
2402	gen_helper_fchs_ST0(tcg_env);
2403	break;
2404	case 1: /* fabs */
2405	gen_helper_fabs_ST0(tcg_env);
2406	break;
2407	case 4: /* ftst */
2408	gen_helper_fldz_FT0(tcg_env);
2409	gen_helper_fcom_ST0_FT0(tcg_env);
2410	break;
2411	case 5: /* fxam */
2412	gen_helper_fxam_ST0(tcg_env);
2413	break;
2414	default:
2415	goto illegal_op;
2416	}
2417	break;
2418	case 0x0d: /* grp d9/5 */
2419	switch (rm) {
2420	case 0:
2421	gen_helper_fpush(tcg_env);
2422	gen_helper_fld1_ST0(tcg_env);
2423	break;
2424	case 1:
2425	gen_helper_fpush(tcg_env);
2426	gen_helper_fldl2t_ST0(tcg_env);
2427	break;
2428	case 2:
2429	gen_helper_fpush(tcg_env);
2430	gen_helper_fldl2e_ST0(tcg_env);
2431	break;
2432	case 3:
2433	gen_helper_fpush(tcg_env);
2434	gen_helper_fldpi_ST0(tcg_env);
2435	break;
2436	case 4:
2437	gen_helper_fpush(tcg_env);
2438	gen_helper_fldlg2_ST0(tcg_env);
2439	break;
2440	case 5:
2441	gen_helper_fpush(tcg_env);
2442	gen_helper_fldln2_ST0(tcg_env);
2443	break;
2444	case 6:
2445	gen_helper_fpush(tcg_env);
2446	gen_helper_fldz_ST0(tcg_env);
2447	break;
2448	default:
2449	goto illegal_op;
2450	}
2451	break;
2452	case 0x0e: /* grp d9/6 */
2453	switch (rm) {
2454	case 0: /* f2xm1 */
2455	gen_helper_f2xm1(tcg_env);
2456	break;
2457	case 1: /* fyl2x */
2458	gen_helper_fyl2x(tcg_env);
2459	break;
2460	case 2: /* fptan */
2461	gen_helper_fptan(tcg_env);
2462	break;
2463	case 3: /* fpatan */
2464	gen_helper_fpatan(tcg_env);
2465	break;
2466	case 4: /* fxtract */
2467	gen_helper_fxtract(tcg_env);
2468	break;
2469	case 5: /* fprem1 */
2470	gen_helper_fprem1(tcg_env);
2471	break;
2472	case 6: /* fdecstp */
2473	gen_helper_fdecstp(tcg_env);
2474	break;
2475	default:
2476	case 7: /* fincstp */
2477	gen_helper_fincstp(tcg_env);
2478	break;
2479	}
2480	break;
2481	case 0x0f: /* grp d9/7 */
2482	switch (rm) {
2483	case 0: /* fprem */
2484	gen_helper_fprem(tcg_env);
2485	break;
2486	case 1: /* fyl2xp1 */
2487	gen_helper_fyl2xp1(tcg_env);
2488	break;
2489	case 2: /* fsqrt */
2490	gen_helper_fsqrt(tcg_env);
2491	break;
2492	case 3: /* fsincos */
2493	gen_helper_fsincos(tcg_env);
2494	break;
2495	case 5: /* fscale */
2496	gen_helper_fscale(tcg_env);
2497	break;
2498	case 4: /* frndint */
2499	gen_helper_frndint(tcg_env);
2500	break;
2501	case 6: /* fsin */
2502	gen_helper_fsin(tcg_env);
2503	break;
2504	default:
2505	case 7: /* fcos */
2506	gen_helper_fcos(tcg_env);
2507	break;
2508	}
2509	break;
2510	case 0x00 ... 0x07: /* fxxx st, sti */
2511	case 0x22 ... 0x23: /* fcom2 and fcomp3, undocumented ops */
2512	case 0x32: /* fcomp5, undocumented op */
2513	gen_helper_fmov_FT0_STN(tcg_env,
2514	tcg_constant_i32(opreg));
2515	gen_helper_fp_arith_ST0_FT0(op & 7);
2516	if (op >= 0x30) {
2517	gen_helper_fpop(tcg_env);
2518	}
2519	break;
2520
2521	case 0x20: case 0x21: case 0x24 ... 0x27: /* fxxx sti, st */
2522	case 0x30: case 0x31: case 0x34 ... 0x37: /* fxxxp sti, st */
2523	gen_helper_fp_arith_STN_ST0(op & 7, opreg);
2524	if (op >= 0x30) {
2525	gen_helper_fpop(tcg_env);
2526	}
2527	break;
2528	case 0x15: /* da/5 */
2529	switch (rm) {
2530	case 1: /* fucompp */
2531	gen_helper_fmov_FT0_STN(tcg_env, tcg_constant_i32(1));
2532	gen_helper_fucom_ST0_FT0(tcg_env);
2533	gen_helper_fpop(tcg_env);
2534	gen_helper_fpop(tcg_env);
2535	break;
2536	default:
2537	goto illegal_op;
2538	}
2539	break;
2540	case 0x1c:
2541	switch (rm) {
2542	case 0: /* feni (287 only, just do nop here) */
2543	break;
2544	case 1: /* fdisi (287 only, just do nop here) */
2545	break;
2546	case 2: /* fclex */
2547	gen_helper_fclex(tcg_env);
2548	update_fip = false;
2549	break;
2550	case 3: /* fninit */
2551	gen_helper_fninit(tcg_env);
2552	update_fip = false;
2553	break;
2554	case 4: /* fsetpm (287 only, just do nop here) */
2555	break;
2556	default:
2557	goto illegal_op;
2558	}
2559	break;
2560	case 0x1d: /* fucomi */
2561	case 0x3d: /* fucomip */
2562	if (!(s->cpuid_features & CPUID_CMOV)) {
2563	goto illegal_op;
2564	}
2565	gen_update_cc_op(s);
2566	gen_helper_fmov_FT0_STN(tcg_env, tcg_constant_i32(opreg));
2567	gen_helper_fucomi_ST0_FT0(tcg_env);
2568	if (op >= 0x30) {
2569	gen_helper_fpop(tcg_env);
2570	}
2571	assume_cc_op(s, CC_OP_EFLAGS);
2572	break;
2573	case 0x1e: /* fcomi */
2574	case 0x3e: /* fcomip */
2575	if (!(s->cpuid_features & CPUID_CMOV)) {
2576	goto illegal_op;
2577	}
2578	gen_update_cc_op(s);
2579	gen_helper_fmov_FT0_STN(tcg_env, tcg_constant_i32(opreg));
2580	gen_helper_fcomi_ST0_FT0(tcg_env);
2581	if (op >= 0x30) {
2582	gen_helper_fpop(tcg_env);
2583	}
2584	assume_cc_op(s, CC_OP_EFLAGS);
2585	break;
2586	case 0x28: /* ffree sti */
2587	case 0x38: /* ffreep sti, undocumented op */
2588	gen_helper_ffree_STN(tcg_env, tcg_constant_i32(opreg));
2589	if (op >= 0x30) {
2590	gen_helper_fpop(tcg_env);
2591	}
2592	break;
2593	case 0x2a: /* fst sti */
2594	case 0x2b: /* fstp sti */
2595	case 0x0b: /* fstp1 sti, undocumented op */
2596	case 0x3a: /* fstp8 sti, undocumented op */
2597	case 0x3b: /* fstp9 sti, undocumented op */
2598	gen_helper_fmov_STN_ST0(tcg_env, tcg_constant_i32(opreg));
2599	if (op != 0x2a) {
2600	gen_helper_fpop(tcg_env);
2601	}
2602	break;
2603	case 0x2c: /* fucom st(i) */
2604	case 0x2d: /* fucomp st(i) */
2605	gen_helper_fmov_FT0_STN(tcg_env, tcg_constant_i32(opreg));
2606	gen_helper_fucom_ST0_FT0(tcg_env);
2607	if (op == 0x2d) {
2608	gen_helper_fpop(tcg_env);
2609	}
2610	break;
2611	case 0x33: /* de/3 */
2612	/*
2613	* TODO: does 0x32 also have the same limitation of requiring
2614	* rm == 1? If so, worth bundling it here and switch the fcom
2615	* helper to gen_helper_fp_arith_ST0_FT0(op & 7).
2616	*/
2617	switch (rm) {
2618	case 1: /* fcompp */
2619	gen_helper_fmov_FT0_STN(tcg_env, tcg_constant_i32(1));
2620	gen_helper_fcom_ST0_FT0(tcg_env);
2621	gen_helper_fpop(tcg_env);
2622	gen_helper_fpop(tcg_env);
2623	break;
2624	default:
2625	goto illegal_op;
2626	}
2627	break;
2628	case 0x3c: /* df/4 */
2629	switch (rm) {
2630	case 0:
2631	t32 = tcg_temp_new_i32();
2632	gen_helper_fnstsw(t32, tcg_env);
2633	tcg_gen_extu_i32_tl(s->T0, t32);
2634	gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
2635	break;
2636	default:
2637	goto illegal_op;
2638	}
2639	break;
2640	case 0x10 ... 0x13: /* fcmovxx */
2641	case 0x18 ... 0x1b:
2642	{
2643	int op1;
2644	TCGLabel *l1;
2645	static const uint8_t fcmov_cc[8] = {
2646	(JCC_B << 1),
2647	(JCC_Z << 1),
2648	(JCC_BE << 1),
2649	(JCC_P << 1),
2650	};
2651
2652	if (!(s->cpuid_features & CPUID_CMOV)) {
2653	goto illegal_op;
2654	}
2655	op1 = fcmov_cc[op & 3] \| (((op >> 3) & 1) ^ 1);
2656	l1 = gen_new_label();
2657	gen_jcc_noeob(s, op1, l1);
2658	gen_helper_fmov_ST0_STN(tcg_env,
2659	tcg_constant_i32(opreg));
2660	gen_set_label(l1);
2661	}
2662	break;
2663	default:
2664	goto illegal_op;
2665	}
2666	}
2667
2668	if (update_fip) {
2669	TCGv_i32 t32 = tcg_temp_new_i32();
2670	tcg_gen_ld_i32(t32, tcg_env,
2671	offsetof(CPUX86State, segs[R_CS].selector));
2672	tcg_gen_st16_i32(t32, tcg_env,
2673	offsetof(CPUX86State, fpcs));
2674	tcg_gen_st_tl(eip_cur_tl(s),
2675	tcg_env, offsetof(CPUX86State, fpip));
2676	}
2677	return;
2678
2679	illegal_op:
2680	gen_illegal_opcode(s);
2681	}
2682
2683	static void gen_multi0F(DisasContext s, X86DecodedInsn decode)
2684	{
2685	int prefixes = s->prefix;
2686	MemOp dflag = s->dflag;
2687	int b = decode->b + 0x100;
2688	int modrm = s->modrm;
2689	MemOp ot;
2690	int reg, rm, mod, op;
2691	TCGv_i32 t32;
2692	TCGv_i64 t64;
2693
2694	/* now check op code */
2695	switch (b) {
2696	case 0x1c7: /* RDSEED, RDPID with f3 prefix */
2697	mod = (modrm >> 6) & 3;
2698	switch ((modrm >> 3) & 7) {
2699	case 7:
2700	if (mod != 3 \|\|
2701	(s->prefix & PREFIX_REPNZ)) {
2702	goto illegal_op;
2703	}
2704	if (s->prefix & PREFIX_REPZ) {
2705	if (!(s->cpuid_7_0_ecx_features & CPUID_7_0_ECX_RDPID)) {
2706	goto illegal_op;
2707	}
2708	gen_helper_rdpid(s->T0, tcg_env);
2709	rm = (modrm & 7) \| REX_B(s);
2710	gen_op_mov_reg_v(s, dflag, rm, s->T0);
2711	break;
2712	} else {
2713	if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_RDSEED)) {
2714	goto illegal_op;
2715	}
2716	goto do_rdrand;
2717	}
2718
2719	case 6: /* RDRAND */
2720	if (mod != 3 \|\|
2721	(s->prefix & (PREFIX_REPZ \| PREFIX_REPNZ)) \|\|
2722	!(s->cpuid_ext_features & CPUID_EXT_RDRAND)) {
2723	goto illegal_op;
2724	}
2725	do_rdrand:
2726	translator_io_start(&s->base);
2727	gen_helper_rdrand(s->T0, tcg_env);
2728	rm = (modrm & 7) \| REX_B(s);
2729	gen_op_mov_reg_v(s, dflag, rm, s->T0);
2730	assume_cc_op(s, CC_OP_EFLAGS);
2731	break;
2732
2733	default:
2734	goto illegal_op;
2735	}
2736	break;
2737
2738	case 0x100:
2739	mod = (modrm >> 6) & 3;
2740	op = (modrm >> 3) & 7;
2741	switch(op) {
2742	case 0: /* sldt */
2743	if (!PE(s) \|\| VM86(s))
2744	goto illegal_op;
2745	if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
2746	break;
2747	}
2748	gen_svm_check_intercept(s, SVM_EXIT_LDTR_READ);
2749	tcg_gen_ld32u_tl(s->T0, tcg_env,
2750	offsetof(CPUX86State, ldt.selector));
2751	ot = mod == 3 ? dflag : MO_16;
2752	gen_st_modrm(s, decode, ot);
2753	break;
2754	case 2: /* lldt */
2755	if (!PE(s) \|\| VM86(s))
2756	goto illegal_op;
2757	if (check_cpl0(s)) {
2758	t32 = tcg_temp_new_i32();
2759	gen_svm_check_intercept(s, SVM_EXIT_LDTR_WRITE);
2760	gen_ld_modrm(s, decode, MO_16);
2761	tcg_gen_trunc_tl_i32(t32, s->T0);
2762	gen_helper_lldt(tcg_env, t32);
2763	}
2764	break;
2765	case 1: /* str */
2766	if (!PE(s) \|\| VM86(s))
2767	goto illegal_op;
2768	if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
2769	break;
2770	}
2771	gen_svm_check_intercept(s, SVM_EXIT_TR_READ);
2772	tcg_gen_ld32u_tl(s->T0, tcg_env,
2773	offsetof(CPUX86State, tr.selector));
2774	ot = mod == 3 ? dflag : MO_16;
2775	gen_st_modrm(s, decode, ot);
2776	break;
2777	case 3: /* ltr */
2778	if (!PE(s) \|\| VM86(s))
2779	goto illegal_op;
2780	if (check_cpl0(s)) {
2781	t32 = tcg_temp_new_i32();
2782	gen_svm_check_intercept(s, SVM_EXIT_TR_WRITE);
2783	gen_ld_modrm(s, decode, MO_16);
2784	tcg_gen_trunc_tl_i32(t32, s->T0);
2785	gen_helper_ltr(tcg_env, t32);
2786	}
2787	break;
2788	case 4: /* verr */
2789	case 5: /* verw */
2790	if (!PE(s) \|\| VM86(s))
2791	goto illegal_op;
2792	gen_ld_modrm(s, decode, MO_16);
2793	gen_update_cc_op(s);
2794	if (op == 4) {
2795	gen_helper_verr(tcg_env, s->T0);
2796	} else {
2797	gen_helper_verw(tcg_env, s->T0);
2798	}
2799	assume_cc_op(s, CC_OP_EFLAGS);
2800	break;
2801	default:
2802	goto illegal_op;
2803	}
2804	break;
2805
2806	case 0x101:
2807	switch (modrm) {
2808	CASE_MODRM_MEM_OP(0): /* sgdt */
2809	if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
2810	break;
2811	}
2812	gen_svm_check_intercept(s, SVM_EXIT_GDTR_READ);
2813	gen_lea_modrm(s, decode);
2814	tcg_gen_ld32u_tl(s->T0,
2815	tcg_env, offsetof(CPUX86State, gdt.limit));
2816	gen_op_st_v(s, MO_16, s->T0, s->A0);
2817	gen_add_A0_im(s, 2);
2818	tcg_gen_ld_tl(s->T0, tcg_env, offsetof(CPUX86State, gdt.base));
2819	/*
2820	* NB: Despite a confusing description in Intel CPU documentation,
2821	* all 32-bits are written regardless of operand size.
2822	*/
2823	gen_op_st_v(s, CODE64(s) + MO_32, s->T0, s->A0);
2824	break;
2825
2826	case 0xc8: /* monitor */
2827	if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) \|\| CPL(s) != 0) {
2828	goto illegal_op;
2829	}
2830	gen_update_cc_op(s);
2831	gen_update_eip_cur(s);
2832	gen_lea_v_seg(s, cpu_regs[R_EAX], R_DS, s->override);
2833	gen_helper_monitor(tcg_env, s->A0);
2834	break;
2835
2836	case 0xc9: /* mwait */
2837	if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) \|\| CPL(s) != 0) {
2838	goto illegal_op;
2839	}
2840	gen_update_cc_op(s);
2841	gen_update_eip_cur(s);
2842	gen_helper_mwait(tcg_env, cur_insn_len_i32(s));
2843	s->base.is_jmp = DISAS_NORETURN;
2844	break;
2845
2846	case 0xca: /* clac */
2847	if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)
2848	\|\| CPL(s) != 0) {
2849	goto illegal_op;
2850	}
2851	gen_reset_eflags(s, AC_MASK);
2852	s->base.is_jmp = DISAS_EOB_NEXT;
2853	break;
2854
2855	case 0xcb: /* stac */
2856	if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)
2857	\|\| CPL(s) != 0) {
2858	goto illegal_op;
2859	}
2860	gen_set_eflags(s, AC_MASK);
2861	s->base.is_jmp = DISAS_EOB_NEXT;
2862	break;
2863
2864	CASE_MODRM_MEM_OP(1): /* sidt */
2865	if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
2866	break;
2867	}
2868	gen_svm_check_intercept(s, SVM_EXIT_IDTR_READ);
2869	gen_lea_modrm(s, decode);
2870	tcg_gen_ld32u_tl(s->T0, tcg_env, offsetof(CPUX86State, idt.limit));
2871	gen_op_st_v(s, MO_16, s->T0, s->A0);
2872	gen_add_A0_im(s, 2);
2873	tcg_gen_ld_tl(s->T0, tcg_env, offsetof(CPUX86State, idt.base));
2874	/*
2875	* NB: Despite a confusing description in Intel CPU documentation,
2876	* all 32-bits are written regardless of operand size.
2877	*/
2878	gen_op_st_v(s, CODE64(s) + MO_32, s->T0, s->A0);
2879	break;
2880
2881	case 0xd0: /* xgetbv */
2882	if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
2883	\|\| (s->prefix & (PREFIX_DATA \| PREFIX_REPZ \| PREFIX_REPNZ))) {
2884	goto illegal_op;
2885	}
2886	t32 = tcg_temp_new_i32();
2887	t64 = tcg_temp_new_i64();
2888	tcg_gen_trunc_tl_i32(t32, cpu_regs[R_ECX]);
2889	gen_helper_xgetbv(t64, tcg_env, t32);
2890	tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], t64);
2891	break;
2892
2893	case 0xd1: /* xsetbv */
2894	if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
2895	\|\| (s->prefix & (PREFIX_DATA \| PREFIX_REPZ \| PREFIX_REPNZ))) {
2896	goto illegal_op;
2897	}
2898	gen_svm_check_intercept(s, SVM_EXIT_XSETBV);
2899	if (!check_cpl0(s)) {
2900	break;
2901	}
2902	t32 = tcg_temp_new_i32();
2903	t64 = tcg_temp_new_i64();
2904	tcg_gen_concat_tl_i64(t64, cpu_regs[R_EAX],
2905	cpu_regs[R_EDX]);
2906	tcg_gen_trunc_tl_i32(t32, cpu_regs[R_ECX]);
2907	gen_helper_xsetbv(tcg_env, t32, t64);
2908	/* End TB because translation flags may change. */
2909	s->base.is_jmp = DISAS_EOB_NEXT;
2910	break;
2911
2912	case 0xd8: /* VMRUN */
2913	if (!SVME(s) \|\| !PE(s)) {
2914	goto illegal_op;
2915	}
2916	if (!check_cpl0(s)) {
2917	break;
2918	}
2919	gen_update_cc_op(s);
2920	gen_update_eip_cur(s);
2921	/*
2922	* Reloads INHIBIT_IRQ mask as well as TF and RF with guest state.
2923	* The usual gen_eob() handling is performed on vmexit after
2924	* host state is reloaded.
2925	*/
2926	gen_helper_vmrun(tcg_env, tcg_constant_i32(s->aflag - 1),
2927	cur_insn_len_i32(s));
2928	tcg_gen_exit_tb(NULL, 0);
2929	s->base.is_jmp = DISAS_NORETURN;
2930	break;
2931
2932	case 0xd9: /* VMMCALL */
2933	if (!SVME(s)) {
2934	goto illegal_op;
2935	}
2936	gen_update_cc_op(s);
2937	gen_update_eip_cur(s);
2938	gen_helper_vmmcall(tcg_env);
2939	break;
2940
2941	case 0xda: /* VMLOAD */
2942	if (!SVME(s) \|\| !PE(s)) {
2943	goto illegal_op;
2944	}
2945	if (!check_cpl0(s)) {
2946	break;
2947	}
2948	gen_update_cc_op(s);
2949	gen_update_eip_cur(s);
2950	gen_helper_vmload(tcg_env, tcg_constant_i32(s->aflag - 1));
2951	break;
2952
2953	case 0xdb: /* VMSAVE */
2954	if (!SVME(s) \|\| !PE(s)) {
2955	goto illegal_op;
2956	}
2957	if (!check_cpl0(s)) {
2958	break;
2959	}
2960	gen_update_cc_op(s);
2961	gen_update_eip_cur(s);
2962	gen_helper_vmsave(tcg_env, tcg_constant_i32(s->aflag - 1));
2963	break;
2964
2965	case 0xdc: /* STGI */
2966	if ((!SVME(s) && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
2967	\|\| !PE(s)) {
2968	goto illegal_op;
2969	}
2970	if (!check_cpl0(s)) {
2971	break;
2972	}
2973	gen_update_cc_op(s);
2974	gen_helper_stgi(tcg_env);
2975	s->base.is_jmp = DISAS_EOB_NEXT;
2976	break;
2977
2978	case 0xdd: /* CLGI */
2979	if (!SVME(s) \|\| !PE(s)) {
2980	goto illegal_op;
2981	}
2982	if (!check_cpl0(s)) {
2983	break;
2984	}
2985	gen_update_cc_op(s);
2986	gen_update_eip_cur(s);
2987	gen_helper_clgi(tcg_env);
2988	break;
2989
2990	case 0xde: /* SKINIT */
2991	if ((!SVME(s) && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
2992	\|\| !PE(s)) {
2993	goto illegal_op;
2994	}
2995	gen_svm_check_intercept(s, SVM_EXIT_SKINIT);
2996	/* If not intercepted, not implemented -- raise #UD. */
2997	goto illegal_op;
2998
2999	case 0xdf: /* INVLPGA */
3000	if (!SVME(s) \|\| !PE(s)) {
3001	goto illegal_op;
3002	}
3003	if (!check_cpl0(s)) {
3004	break;
3005	}
3006	gen_svm_check_intercept(s, SVM_EXIT_INVLPGA);
3007	if (s->aflag == MO_64) {
3008	tcg_gen_mov_tl(s->A0, cpu_regs[R_EAX]);
3009	} else {
3010	tcg_gen_ext32u_tl(s->A0, cpu_regs[R_EAX]);
3011	}
3012	gen_helper_flush_page(tcg_env, s->A0);
3013	s->base.is_jmp = DISAS_EOB_NEXT;
3014	break;
3015
3016	CASE_MODRM_MEM_OP(2): /* lgdt */
3017	if (!check_cpl0(s)) {
3018	break;
3019	}
3020	gen_svm_check_intercept(s, SVM_EXIT_GDTR_WRITE);
3021	gen_lea_modrm(s, decode);
3022	gen_op_ld_v(s, MO_16, s->T1, s->A0);
3023	gen_add_A0_im(s, 2);
3024	gen_op_ld_v(s, CODE64(s) + MO_32, s->T0, s->A0);
3025	if (dflag == MO_16) {
3026	tcg_gen_andi_tl(s->T0, s->T0, 0xffffff);
3027	}
3028	tcg_gen_st_tl(s->T0, tcg_env, offsetof(CPUX86State, gdt.base));
3029	tcg_gen_st32_tl(s->T1, tcg_env, offsetof(CPUX86State, gdt.limit));
3030	break;
3031
3032	CASE_MODRM_MEM_OP(3): /* lidt */
3033	if (!check_cpl0(s)) {
3034	break;
3035	}
3036	gen_svm_check_intercept(s, SVM_EXIT_IDTR_WRITE);
3037	gen_lea_modrm(s, decode);
3038	gen_op_ld_v(s, MO_16, s->T1, s->A0);
3039	gen_add_A0_im(s, 2);
3040	gen_op_ld_v(s, CODE64(s) + MO_32, s->T0, s->A0);
3041	if (dflag == MO_16) {
3042	tcg_gen_andi_tl(s->T0, s->T0, 0xffffff);
3043	}
3044	tcg_gen_st_tl(s->T0, tcg_env, offsetof(CPUX86State, idt.base));
3045	tcg_gen_st32_tl(s->T1, tcg_env, offsetof(CPUX86State, idt.limit));
3046	break;
3047
3048	CASE_MODRM_OP(4): /* smsw */
3049	if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
3050	break;
3051	}
3052	gen_svm_check_intercept(s, SVM_EXIT_READ_CR0);
3053	tcg_gen_ld_tl(s->T0, tcg_env, offsetof(CPUX86State, cr[0]));
3054	/*
3055	* In 32-bit mode, the higher 16 bits of the destination
3056	* register are undefined. In practice CR0[31:0] is stored
3057	* just like in 64-bit mode.
3058	*/
3059	mod = (modrm >> 6) & 3;
3060	ot = (mod != 3 ? MO_16 : s->dflag);
3061	gen_st_modrm(s, decode, ot);
3062	break;
3063	case 0xee: /* rdpkru */
3064	if (s->prefix & (PREFIX_DATA \| PREFIX_REPZ \| PREFIX_REPNZ)) {
3065	goto illegal_op;
3066	}
3067	t32 = tcg_temp_new_i32();
3068	t64 = tcg_temp_new_i64();
3069	tcg_gen_trunc_tl_i32(t32, cpu_regs[R_ECX]);
3070	gen_helper_rdpkru(t64, tcg_env, t32);
3071	tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], t64);
3072	break;
3073	case 0xef: /* wrpkru */
3074	if (s->prefix & (PREFIX_DATA \| PREFIX_REPZ \| PREFIX_REPNZ)) {
3075	goto illegal_op;
3076	}
3077	t32 = tcg_temp_new_i32();
3078	t64 = tcg_temp_new_i64();
3079	tcg_gen_concat_tl_i64(t64, cpu_regs[R_EAX],
3080	cpu_regs[R_EDX]);
3081	tcg_gen_trunc_tl_i32(t32, cpu_regs[R_ECX]);
3082	gen_helper_wrpkru(tcg_env, t32, t64);
3083	break;
3084
3085	CASE_MODRM_OP(6): /* lmsw */
3086	if (!check_cpl0(s)) {
3087	break;
3088	}
3089	gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0);
3090	gen_ld_modrm(s, decode, MO_16);
3091	/*
3092	* Only the 4 lower bits of CR0 are modified.
3093	* PE cannot be set to zero if already set to one.
3094	*/
3095	tcg_gen_ld_tl(s->T1, tcg_env, offsetof(CPUX86State, cr[0]));
3096	tcg_gen_andi_tl(s->T0, s->T0, 0xf);
3097	tcg_gen_andi_tl(s->T1, s->T1, ~0xe);
3098	tcg_gen_or_tl(s->T0, s->T0, s->T1);
3099	gen_helper_write_crN(tcg_env, tcg_constant_i32(0), s->T0);
3100	s->base.is_jmp = DISAS_EOB_NEXT;
3101	break;
3102
3103	CASE_MODRM_MEM_OP(7): /* invlpg */
3104	if (!check_cpl0(s)) {
3105	break;
3106	}
3107	gen_svm_check_intercept(s, SVM_EXIT_INVLPG);
3108	gen_lea_modrm(s, decode);
3109	gen_helper_flush_page(tcg_env, s->A0);
3110	s->base.is_jmp = DISAS_EOB_NEXT;
3111	break;
3112
3113	case 0xf8: /* swapgs */
3114	#ifdef TARGET_X86_64
3115	if (CODE64(s)) {
3116	if (check_cpl0(s)) {
3117	tcg_gen_mov_tl(s->T0, cpu_seg_base[R_GS]);
3118	tcg_gen_ld_tl(cpu_seg_base[R_GS], tcg_env,
3119	offsetof(CPUX86State, kernelgsbase));
3120	tcg_gen_st_tl(s->T0, tcg_env,
3121	offsetof(CPUX86State, kernelgsbase));
3122	}
3123	break;
3124	}
3125	#endif
3126	goto illegal_op;
3127
3128	case 0xf9: /* rdtscp */
3129	if (!(s->cpuid_ext2_features & CPUID_EXT2_RDTSCP)) {
3130	goto illegal_op;
3131	}
3132	gen_update_cc_op(s);
3133	gen_update_eip_cur(s);
3134	translator_io_start(&s->base);
3135	gen_helper_rdtsc(tcg_env);
3136	gen_helper_rdpid(s->T0, tcg_env);
3137	gen_op_mov_reg_v(s, dflag, R_ECX, s->T0);
3138	break;
3139
3140	default:
3141	goto illegal_op;
3142	}
3143	break;
3144
3145	case 0x11a:
3146	if (s->flags & HF_MPX_EN_MASK) {
3147	mod = (modrm >> 6) & 3;
3148	reg = ((modrm >> 3) & 7) \| REX_R(s);
3149	if (prefixes & PREFIX_REPZ) {
3150	/* bndcl */
3151	if (reg >= 4
3152	\|\| s->aflag == MO_16) {
3153	goto illegal_op;
3154	}
3155	gen_bndck(s, decode, TCG_COND_LTU, cpu_bndl[reg]);
3156	} else if (prefixes & PREFIX_REPNZ) {
3157	/* bndcu */
3158	if (reg >= 4
3159	\|\| s->aflag == MO_16) {
3160	goto illegal_op;
3161	}
3162	TCGv_i64 notu = tcg_temp_new_i64();
3163	tcg_gen_not_i64(notu, cpu_bndu[reg]);
3164	gen_bndck(s, decode, TCG_COND_GTU, notu);
3165	} else if (prefixes & PREFIX_DATA) {
3166	/* bndmov -- from reg/mem */
3167	if (reg >= 4 \|\| s->aflag == MO_16) {
3168	goto illegal_op;
3169	}
3170	if (mod == 3) {
3171	int reg2 = (modrm & 7) \| REX_B(s);
3172	if (reg2 >= 4) {
3173	goto illegal_op;
3174	}
3175	if (s->flags & HF_MPX_IU_MASK) {
3176	tcg_gen_mov_i64(cpu_bndl[reg], cpu_bndl[reg2]);
3177	tcg_gen_mov_i64(cpu_bndu[reg], cpu_bndu[reg2]);
3178	}
3179	} else {
3180	gen_lea_modrm(s, decode);
3181	if (CODE64(s)) {
3182	tcg_gen_qemu_ld_i64(cpu_bndl[reg], s->A0,
3183	s->mem_index, MO_LEUQ);
3184	tcg_gen_addi_tl(s->A0, s->A0, 8);
3185	tcg_gen_qemu_ld_i64(cpu_bndu[reg], s->A0,
3186	s->mem_index, MO_LEUQ);
3187	} else {
3188	tcg_gen_qemu_ld_i64(cpu_bndl[reg], s->A0,
3189	s->mem_index, MO_LEUL);
3190	tcg_gen_addi_tl(s->A0, s->A0, 4);
3191	tcg_gen_qemu_ld_i64(cpu_bndu[reg], s->A0,
3192	s->mem_index, MO_LEUL);
3193	}
3194	/* bnd registers are now in-use */
3195	gen_set_hflag(s, HF_MPX_IU_MASK);
3196	}
3197	} else if (mod != 3) {
3198	/* bndldx */
3199	AddressParts a = decode->mem;
3200	if (reg >= 4
3201	\|\| s->aflag == MO_16
3202	\|\| a.base < -1) {
3203	goto illegal_op;
3204	}
3205	if (a.base >= 0) {
3206	tcg_gen_addi_tl(s->A0, cpu_regs[a.base], a.disp);
3207	} else {
3208	tcg_gen_movi_tl(s->A0, 0);
3209	}
3210	gen_lea_v_seg(s, s->A0, a.def_seg, s->override);
3211	if (a.index >= 0) {
3212	tcg_gen_mov_tl(s->T0, cpu_regs[a.index]);
3213	} else {
3214	tcg_gen_movi_tl(s->T0, 0);
3215	}
3216	if (CODE64(s)) {
3217	gen_helper_bndldx64(cpu_bndl[reg], tcg_env, s->A0, s->T0);
3218	tcg_gen_ld_i64(cpu_bndu[reg], tcg_env,
3219	offsetof(CPUX86State, mmx_t0.MMX_Q(0)));
3220	} else {
3221	gen_helper_bndldx32(cpu_bndu[reg], tcg_env, s->A0, s->T0);
3222	tcg_gen_ext32u_i64(cpu_bndl[reg], cpu_bndu[reg]);
3223	tcg_gen_shri_i64(cpu_bndu[reg], cpu_bndu[reg], 32);
3224	}
3225	gen_set_hflag(s, HF_MPX_IU_MASK);
3226	}
3227	}
3228	break;
3229	case 0x11b:
3230	if (s->flags & HF_MPX_EN_MASK) {
3231	mod = (modrm >> 6) & 3;
3232	reg = ((modrm >> 3) & 7) \| REX_R(s);
3233	if (mod != 3 && (prefixes & PREFIX_REPZ)) {
3234	/* bndmk */
3235	if (reg >= 4
3236	\|\| s->aflag == MO_16) {
3237	goto illegal_op;
3238	}
3239	AddressParts a = decode->mem;
3240	if (a.base >= 0) {
3241	tcg_gen_extu_tl_i64(cpu_bndl[reg], cpu_regs[a.base]);
3242	if (!CODE64(s)) {
3243	tcg_gen_ext32u_i64(cpu_bndl[reg], cpu_bndl[reg]);
3244	}
3245	} else if (a.base == -1) {
3246	/* no base register has lower bound of 0 */
3247	tcg_gen_movi_i64(cpu_bndl[reg], 0);
3248	} else {
3249	/* rip-relative generates #ud */
3250	goto illegal_op;
3251	}
3252	tcg_gen_not_tl(s->A0, gen_lea_modrm_1(s, decode->mem, false));
3253	if (!CODE64(s)) {
3254	tcg_gen_ext32u_tl(s->A0, s->A0);
3255	}
3256	tcg_gen_extu_tl_i64(cpu_bndu[reg], s->A0);
3257	/* bnd registers are now in-use */
3258	gen_set_hflag(s, HF_MPX_IU_MASK);
3259	break;
3260	} else if (prefixes & PREFIX_REPNZ) {
3261	/* bndcn */
3262	if (reg >= 4
3263	\|\| s->aflag == MO_16) {
3264	goto illegal_op;
3265	}
3266	gen_bndck(s, decode, TCG_COND_GTU, cpu_bndu[reg]);
3267	} else if (prefixes & PREFIX_DATA) {
3268	/* bndmov -- to reg/mem */
3269	if (reg >= 4 \|\| s->aflag == MO_16) {
3270	goto illegal_op;
3271	}
3272	if (mod == 3) {
3273	int reg2 = (modrm & 7) \| REX_B(s);
3274	if (reg2 >= 4) {
3275	goto illegal_op;
3276	}
3277	if (s->flags & HF_MPX_IU_MASK) {
3278	tcg_gen_mov_i64(cpu_bndl[reg2], cpu_bndl[reg]);
3279	tcg_gen_mov_i64(cpu_bndu[reg2], cpu_bndu[reg]);
3280	}
3281	} else {
3282	gen_lea_modrm(s, decode);
3283	if (CODE64(s)) {
3284	tcg_gen_qemu_st_i64(cpu_bndl[reg], s->A0,
3285	s->mem_index, MO_LEUQ);
3286	tcg_gen_addi_tl(s->A0, s->A0, 8);
3287	tcg_gen_qemu_st_i64(cpu_bndu[reg], s->A0,
3288	s->mem_index, MO_LEUQ);
3289	} else {
3290	tcg_gen_qemu_st_i64(cpu_bndl[reg], s->A0,
3291	s->mem_index, MO_LEUL);
3292	tcg_gen_addi_tl(s->A0, s->A0, 4);
3293	tcg_gen_qemu_st_i64(cpu_bndu[reg], s->A0,
3294	s->mem_index, MO_LEUL);
3295	}
3296	}
3297	} else if (mod != 3) {
3298	/* bndstx */
3299	AddressParts a = decode->mem;
3300	if (reg >= 4
3301	\|\| s->aflag == MO_16
3302	\|\| a.base < -1) {
3303	goto illegal_op;
3304	}
3305	if (a.base >= 0) {
3306	tcg_gen_addi_tl(s->A0, cpu_regs[a.base], a.disp);
3307	} else {
3308	tcg_gen_movi_tl(s->A0, 0);
3309	}
3310	gen_lea_v_seg(s, s->A0, a.def_seg, s->override);
3311	if (a.index >= 0) {
3312	tcg_gen_mov_tl(s->T0, cpu_regs[a.index]);
3313	} else {
3314	tcg_gen_movi_tl(s->T0, 0);
3315	}
3316	if (CODE64(s)) {
3317	gen_helper_bndstx64(tcg_env, s->A0, s->T0,
3318	cpu_bndl[reg], cpu_bndu[reg]);
3319	} else {
3320	gen_helper_bndstx32(tcg_env, s->A0, s->T0,
3321	cpu_bndl[reg], cpu_bndu[reg]);
3322	}
3323	}
3324	}
3325	break;
3326	default:
3327	g_assert_not_reached();
3328	}
3329	return;
3330	illegal_op:
3331	gen_illegal_opcode(s);
3332	}
3333
3334	#include "decode-new.c.inc"
3335
3336	void tcg_x86_init(void)
3337	{
3338	static const char reg_names[CPU_NB_REGS][4] = {
3339	#ifdef TARGET_X86_64
3340	[R_EAX] = "rax",
3341	[R_EBX] = "rbx",
3342	[R_ECX] = "rcx",
3343	[R_EDX] = "rdx",
3344	[R_ESI] = "rsi",
3345	[R_EDI] = "rdi",
3346	[R_EBP] = "rbp",
3347	[R_ESP] = "rsp",
3348	[8] = "r8",
3349	[9] = "r9",
3350	[10] = "r10",
3351	[11] = "r11",
3352	[12] = "r12",
3353	[13] = "r13",
3354	[14] = "r14",
3355	[15] = "r15",
3356	#else
3357	[R_EAX] = "eax",
3358	[R_EBX] = "ebx",
3359	[R_ECX] = "ecx",
3360	[R_EDX] = "edx",
3361	[R_ESI] = "esi",
3362	[R_EDI] = "edi",
3363	[R_EBP] = "ebp",
3364	[R_ESP] = "esp",
3365	#endif
3366	};
3367	static const char eip_name[] = {
3368	#ifdef TARGET_X86_64
3369	"rip"
3370	#else
3371	"eip"
3372	#endif
3373	};
3374	static const char seg_base_names[6][8] = {
3375	[R_CS] = "cs_base",
3376	[R_DS] = "ds_base",
3377	[R_ES] = "es_base",
3378	[R_FS] = "fs_base",
3379	[R_GS] = "gs_base",
3380	[R_SS] = "ss_base",
3381	};
3382	static const char bnd_regl_names[4][8] = {
3383	"bnd0_lb", "bnd1_lb", "bnd2_lb", "bnd3_lb"
3384	};
3385	static const char bnd_regu_names[4][8] = {
3386	"bnd0_ub", "bnd1_ub", "bnd2_ub", "bnd3_ub"
3387	};
3388	int i;
3389
3390	cpu_cc_op = tcg_global_mem_new_i32(tcg_env,
3391	offsetof(CPUX86State, cc_op), "cc_op");
3392	cpu_cc_dst = tcg_global_mem_new(tcg_env, offsetof(CPUX86State, cc_dst),
3393	"cc_dst");
3394	cpu_cc_src = tcg_global_mem_new(tcg_env, offsetof(CPUX86State, cc_src),
3395	"cc_src");
3396	cpu_cc_src2 = tcg_global_mem_new(tcg_env, offsetof(CPUX86State, cc_src2),
3397	"cc_src2");
3398	cpu_eip = tcg_global_mem_new(tcg_env, offsetof(CPUX86State, eip), eip_name);
3399
3400	for (i = 0; i < CPU_NB_REGS; ++i) {
3401	cpu_regs[i] = tcg_global_mem_new(tcg_env,
3402	offsetof(CPUX86State, regs[i]),
3403	reg_names[i]);
3404	}
3405
3406	for (i = 0; i < 6; ++i) {
3407	cpu_seg_base[i]
3408	= tcg_global_mem_new(tcg_env,
3409	offsetof(CPUX86State, segs[i].base),
3410	seg_base_names[i]);
3411	}
3412
3413	for (i = 0; i < 4; ++i) {
3414	cpu_bndl[i]
3415	= tcg_global_mem_new_i64(tcg_env,
3416	offsetof(CPUX86State, bnd_regs[i].lb),
3417	bnd_regl_names[i]);
3418	cpu_bndu[i]
3419	= tcg_global_mem_new_i64(tcg_env,
3420	offsetof(CPUX86State, bnd_regs[i].ub),
3421	bnd_regu_names[i]);
3422	}
3423	}
3424
3425	static void i386_tr_init_disas_context(DisasContextBase dcbase, CPUState cpu)
3426	{
3427	DisasContext *dc = container_of(dcbase, DisasContext, base);
3428	CPUX86State *env = cpu_env(cpu);
3429	uint32_t flags = dc->base.tb->flags;
3430	uint32_t cflags = tb_cflags(dc->base.tb);
3431	int cpl = (flags >> HF_CPL_SHIFT) & 3;
3432	int iopl = (flags >> IOPL_SHIFT) & 3;
3433
3434	dc->cs_base = dc->base.tb->cs_base;
3435	dc->pc_save = dc->base.pc_next;
3436	dc->flags = flags;
3437	#ifndef CONFIG_USER_ONLY
3438	dc->cpl = cpl;
3439	dc->iopl = iopl;
3440	#endif
3441
3442	/* We make some simplifying assumptions; validate they're correct. */
3443	g_assert(PE(dc) == ((flags & HF_PE_MASK) != 0));
3444	g_assert(CPL(dc) == cpl);
3445	g_assert(IOPL(dc) == iopl);
3446	g_assert(VM86(dc) == ((flags & HF_VM_MASK) != 0));
3447	g_assert(CODE32(dc) == ((flags & HF_CS32_MASK) != 0));
3448	g_assert(CODE64(dc) == ((flags & HF_CS64_MASK) != 0));
3449	g_assert(SS32(dc) == ((flags & HF_SS32_MASK) != 0));
3450	g_assert(LMA(dc) == ((flags & HF_LMA_MASK) != 0));
3451	g_assert(ADDSEG(dc) == ((flags & HF_ADDSEG_MASK) != 0));
3452	g_assert(SVME(dc) == ((flags & HF_SVME_MASK) != 0));
3453	g_assert(GUEST(dc) == ((flags & HF_GUEST_MASK) != 0));
3454
3455	dc->cc_op = CC_OP_DYNAMIC;
3456	dc->cc_op_dirty = false;
3457	/* select memory access functions */
3458	dc->mem_index = cpu_mmu_index(cpu, false);
3459	dc->cpuid_features = env->features[FEAT_1_EDX];
3460	dc->cpuid_ext_features = env->features[FEAT_1_ECX];
3461	dc->cpuid_ext2_features = env->features[FEAT_8000_0001_EDX];
3462	dc->cpuid_ext3_features = env->features[FEAT_8000_0001_ECX];
3463	dc->cpuid_7_0_ebx_features = env->features[FEAT_7_0_EBX];
3464	dc->cpuid_7_0_ecx_features = env->features[FEAT_7_0_ECX];
3465	dc->cpuid_7_1_eax_features = env->features[FEAT_7_1_EAX];
3466	dc->cpuid_xsave_features = env->features[FEAT_XSAVE];
3467	dc->jmp_opt = !((cflags & CF_NO_GOTO_TB) \|\|
3468	(flags & (HF_RF_MASK \| HF_TF_MASK \| HF_INHIBIT_IRQ_MASK)));
3469
3470	dc->T0 = tcg_temp_new();
3471	dc->T1 = tcg_temp_new();
3472	dc->A0 = tcg_temp_new();
3473
3474	dc->cc_srcT = NULL;
3475	}
3476
3477	static void i386_tr_tb_start(DisasContextBase db, CPUState cpu)
3478	{
3479	}
3480
3481	static void i386_tr_insn_start(DisasContextBase dcbase, CPUState cpu)
3482	{
3483	DisasContext *dc = container_of(dcbase, DisasContext, base);
3484	target_ulong pc_arg = dc->base.pc_next;
3485
3486	dc->prev_insn_start = dc->base.insn_start;
3487	dc->prev_insn_end = tcg_last_op();
3488	if (tb_cflags(dcbase->tb) & CF_PCREL) {
3489	pc_arg &= ~TARGET_PAGE_MASK;
3490	}
3491	tcg_gen_insn_start(pc_arg, dc->cc_op, 0);
3492	}
3493
3494	static void i386_tr_translate_insn(DisasContextBase dcbase, CPUState cpu)
3495	{
3496	DisasContext *dc = container_of(dcbase, DisasContext, base);
3497	bool orig_cc_op_dirty = dc->cc_op_dirty;
3498	CCOp orig_cc_op = dc->cc_op;
3499	target_ulong orig_pc_save = dc->pc_save;
3500
3501	#ifdef TARGET_VSYSCALL_PAGE
3502	/*
3503	* Detect entry into the vsyscall page and invoke the syscall.
3504	*/
3505	if ((dc->base.pc_next & TARGET_PAGE_MASK) == TARGET_VSYSCALL_PAGE) {
3506	gen_exception(dc, EXCP_VSYSCALL);
3507	dc->base.pc_next = dc->pc + 1;
3508	return;
3509	}
3510	#endif
3511
3512	switch (sigsetjmp(dc->jmpbuf, 0)) {
3513	case 0:
3514	disas_insn(dc, cpu);
3515	break;
3516	case 1:
3517	gen_exception_gpf(dc);
3518	break;
3519	case 2:
3520	/* Restore state that may affect the next instruction. */
3521	dc->pc = dc->base.pc_next;
3522	assert(dc->cc_op_dirty == orig_cc_op_dirty);
3523	assert(dc->cc_op == orig_cc_op);
3524	assert(dc->pc_save == orig_pc_save);
3525	dc->base.num_insns--;
3526	tcg_remove_ops_after(dc->prev_insn_end);
3527	dc->base.insn_start = dc->prev_insn_start;
3528	dc->base.is_jmp = DISAS_TOO_MANY;
3529	return;
3530	default:
3531	g_assert_not_reached();
3532	}
3533
3534	/*
3535	* Instruction decoding completed (possibly with #GP if the
3536	* 15-byte boundary was exceeded).
3537	*/
3538	dc->base.pc_next = dc->pc;
3539	if (dc->base.is_jmp == DISAS_NEXT) {
3540	if (dc->flags & (HF_TF_MASK \| HF_INHIBIT_IRQ_MASK)) {
3541	/*
3542	* If single step mode, we generate only one instruction and
3543	* generate an exception.
3544	* If irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear
3545	* the flag and abort the translation to give the irqs a
3546	* chance to happen.
3547	*/
3548	dc->base.is_jmp = DISAS_EOB_NEXT;
3549	} else if (!translator_is_same_page(&dc->base, dc->base.pc_next)) {
3550	dc->base.is_jmp = DISAS_TOO_MANY;
3551	}
3552	}
3553	}
3554
3555	static void i386_tr_tb_stop(DisasContextBase dcbase, CPUState cpu)
3556	{
3557	DisasContext *dc = container_of(dcbase, DisasContext, base);
3558
3559	switch (dc->base.is_jmp) {
3560	case DISAS_NORETURN:
3561	/*
3562	* Most instructions should not use DISAS_NORETURN, as that suppresses
3563	* the handling of hflags normally done by gen_eob(). We can
3564	* get here:
3565	* - for exception and interrupts
3566	* - for jump optimization (which is disabled by INHIBIT_IRQ/RF/TF)
3567	* - for VMRUN because RF/TF handling for the host is done after vmexit,
3568	* and INHIBIT_IRQ is loaded from the VMCB
3569	* - for HLT/PAUSE/MWAIT to exit the main loop with specific EXCP_* values;
3570	* the helpers handle themselves the tasks normally done by gen_eob().
3571	*/
3572	break;
3573	case DISAS_TOO_MANY:
3574	gen_update_cc_op(dc);
3575	gen_jmp_rel_csize(dc, 0, 0);
3576	break;
3577	case DISAS_EOB_NEXT:
3578	case DISAS_EOB_INHIBIT_IRQ:
3579	assert(dc->base.pc_next == dc->pc);
3580	gen_update_eip_cur(dc);
3581	/* fall through */
3582	case DISAS_EOB_ONLY:
3583	case DISAS_EOB_RECHECK_TF:
3584	case DISAS_JUMP:
3585	gen_eob(dc, dc->base.is_jmp);
3586	break;
3587	default:
3588	g_assert_not_reached();
3589	}
3590	}
3591
3592	static const TranslatorOps i386_tr_ops = {
3593	.init_disas_context = i386_tr_init_disas_context,
3594	.tb_start = i386_tr_tb_start,
3595	.insn_start = i386_tr_insn_start,
3596	.translate_insn = i386_tr_translate_insn,
3597	.tb_stop = i386_tr_tb_stop,
3598	};
3599
3600	void x86_translate_code(CPUState cpu, TranslationBlock tb,
3601	int max_insns, vaddr pc, void host_pc)
3602	{
3603	DisasContext dc;
3604
3605	translator_loop(cpu, tb, max_insns, pc, host_pc, &i386_tr_ops, &dc.base,
3606	TCG_TYPE_VA);
3607	}