seg_helper.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 2,535 lines 81.7 KB

Raw

1	/*
2	* x86 segmentation related helpers:
3	* TSS, interrupts, system calls, jumps and call/task gates, descriptors
4	*
5	* Copyright (c) 2003 Fabrice Bellard
6	*
7	* This library is free software; you can redistribute it and/or
8	* modify it under the terms of the GNU Lesser General Public
9	* License as published by the Free Software Foundation; either
10	* version 2.1 of the License, or (at your option) any later version.
11	*
12	* This library is distributed in the hope that it will be useful,
13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15	* Lesser General Public License for more details.
16	*
17	* You should have received a copy of the GNU Lesser General Public
18	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
19	*/
20
21	#include "qemu/osdep.h"
22	#include "cpu.h"
23	#include "qemu/log.h"
24	#include "exec/helper-proto.h"
25	#include "accel/tcg/cpu-ldst.h"
26	#include "accel/tcg/probe.h"
27	#include "exec/log.h"
28	#include "helper-tcg.h"
29	#include "seg_helper.h"
30	#include "access.h"
31	#include "tcg-cpu.h"
32	#include "qemu/plugin.h"
33
34	#ifdef TARGET_X86_64
35	#define SET_ESP(val, sp_mask) \
36	do { \
37	if ((sp_mask) == 0xffff) { \
38	env->regs[R_ESP] = (env->regs[R_ESP] & ~0xffff) \| \
39	((val) & 0xffff); \
40	} else if ((sp_mask) == 0xffffffffLL) { \
41	env->regs[R_ESP] = (uint32_t)(val); \
42	} else { \
43	env->regs[R_ESP] = (val); \
44	} \
45	} while (0)
46	#else
47	#define SET_ESP(val, sp_mask) \
48	do { \
49	env->regs[R_ESP] = (env->regs[R_ESP] & ~(sp_mask)) \| \
50	((val) & (sp_mask)); \
51	} while (0)
52	#endif
53
54	/* XXX: use mmu_index to have proper DPL support */
55	typedef struct StackAccess
56	{
57	CPUX86State *env;
58	uintptr_t ra;
59	target_ulong ss_base;
60	target_ulong sp;
61	target_ulong sp_mask;
62	int mmu_index;
63	} StackAccess;
64
65	static void pushw(StackAccess *sa, uint16_t val)
66	{
67	sa->sp -= 2;
68	cpu_stw_le_mmuidx_ra(sa->env, sa->ss_base + (sa->sp & sa->sp_mask),
69	val, sa->mmu_index, sa->ra);
70	}
71
72	static void pushl(StackAccess *sa, uint32_t val)
73	{
74	sa->sp -= 4;
75	cpu_stl_le_mmuidx_ra(sa->env, sa->ss_base + (sa->sp & sa->sp_mask),
76	val, sa->mmu_index, sa->ra);
77	}
78
79	static uint16_t popw(StackAccess *sa)
80	{
81	uint16_t ret = cpu_lduw_le_mmuidx_ra(sa->env,
82	sa->ss_base + (sa->sp & sa->sp_mask),
83	sa->mmu_index, sa->ra);
84	sa->sp += 2;
85	return ret;
86	}
87
88	static uint32_t popl(StackAccess *sa)
89	{
90	uint32_t ret = cpu_ldl_le_mmuidx_ra(sa->env,
91	sa->ss_base + (sa->sp & sa->sp_mask),
92	sa->mmu_index, sa->ra);
93	sa->sp += 4;
94	return ret;
95	}
96
97	int get_pg_mode(CPUX86State *env)
98	{
99	int pg_mode = PG_MODE_PG;
100	if (!(env->cr[0] & CR0_PG_MASK)) {
101	return 0;
102	}
103	if (env->cr[0] & CR0_WP_MASK) {
104	pg_mode \|= PG_MODE_WP;
105	}
106	if (env->cr[4] & CR4_PAE_MASK) {
107	pg_mode \|= PG_MODE_PAE;
108	if (env->efer & MSR_EFER_NXE) {
109	pg_mode \|= PG_MODE_NXE;
110	}
111	}
112	if (env->cr[4] & CR4_PSE_MASK) {
113	pg_mode \|= PG_MODE_PSE;
114	}
115	if (env->cr[4] & CR4_SMEP_MASK) {
116	pg_mode \|= PG_MODE_SMEP;
117	}
118	if (env->hflags & HF_LMA_MASK) {
119	pg_mode \|= PG_MODE_LMA;
120	if (env->cr[4] & CR4_PKE_MASK) {
121	pg_mode \|= PG_MODE_PKE;
122	}
123	if (env->cr[4] & CR4_PKS_MASK) {
124	pg_mode \|= PG_MODE_PKS;
125	}
126	if (env->cr[4] & CR4_LA57_MASK) {
127	pg_mode \|= PG_MODE_LA57;
128	}
129	}
130	return pg_mode;
131	}
132
133	static int x86_mmu_index_kernel_pl(CPUX86State *env, unsigned pl)
134	{
135	int mmu_index_32 = (env->hflags & HF_LMA_MASK) ? 0 : 1;
136	int mmu_index_base =
137	!(env->hflags & HF_SMAP_MASK) ? MMU_KNOSMAP64_IDX :
138	(pl < 3 && (env->eflags & AC_MASK)
139	? MMU_KNOSMAP64_IDX : MMU_KSMAP64_IDX);
140
141	return mmu_index_base + mmu_index_32;
142	}
143
144	int cpu_mmu_index_kernel(CPUX86State *env)
145	{
146	return x86_mmu_index_kernel_pl(env, env->hflags & HF_CPL_MASK);
147	}
148
149	/* return non zero if error */
150	static inline int load_segment_ra(CPUX86State env, uint32_t e1_ptr,
151	uint32_t *e2_ptr, int selector,
152	uintptr_t retaddr)
153	{
154	SegmentCache *dt;
155	int index;
156	target_ulong ptr;
157
158	if (selector & 0x4) {
159	dt = &env->ldt;
160	} else {
161	dt = &env->gdt;
162	}
163	index = selector & ~7;
164	if ((index + 7) > dt->limit) {
165	return -1;
166	}
167	ptr = dt->base + index;
168	*e1_ptr = cpu_ldl_kernel_ra(env, ptr, retaddr);
169	*e2_ptr = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
170	return 0;
171	}
172
173	static inline int load_segment(CPUX86State env, uint32_t e1_ptr,
174	uint32_t *e2_ptr, int selector)
175	{
176	return load_segment_ra(env, e1_ptr, e2_ptr, selector, 0);
177	}
178
179	static inline unsigned int get_seg_limit(uint32_t e1, uint32_t e2)
180	{
181	unsigned int limit;
182
183	limit = (e1 & 0xffff) \| (e2 & 0x000f0000);
184	if (e2 & DESC_G_MASK) {
185	limit = (limit << 12) \| 0xfff;
186	}
187	return limit;
188	}
189
190	static inline uint32_t get_seg_base(uint32_t e1, uint32_t e2)
191	{
192	return (e1 >> 16) \| ((e2 & 0xff) << 16) \| (e2 & 0xff000000);
193	}
194
195	static inline void load_seg_cache_raw_dt(SegmentCache *sc, uint32_t e1,
196	uint32_t e2)
197	{
198	sc->base = get_seg_base(e1, e2);
199	sc->limit = get_seg_limit(e1, e2);
200	sc->flags = e2;
201	}
202
203	/* init the segment cache in vm86 mode. */
204	static inline void load_seg_vm(CPUX86State *env, int seg, int selector)
205	{
206	selector &= 0xffff;
207
208	cpu_x86_load_seg_cache(env, seg, selector, (selector << 4), 0xffff,
209	DESC_P_MASK \| DESC_S_MASK \| DESC_W_MASK \|
210	DESC_A_MASK \| (3 << DESC_DPL_SHIFT));
211	}
212
213	static inline void get_ss_esp_from_tss(CPUX86State env, uint32_t ss_ptr,
214	uint32_t *esp_ptr, int dpl,
215	uintptr_t retaddr)
216	{
217	X86CPU *cpu = env_archcpu(env);
218	int type, index, shift;
219
220	#if 0
221	{
222	int i;
223	printf("TR: base=%p limit=%x\n", env->tr.base, env->tr.limit);
224	for (i = 0; i < env->tr.limit; i++) {
225	printf("%02x ", env->tr.base[i]);
226	if ((i & 7) == 7) {
227	printf("\n");
228	}
229	}
230	printf("\n");
231	}
232	#endif
233
234	if (!(env->tr.flags & DESC_P_MASK)) {
235	cpu_abort(CPU(cpu), "invalid tss");
236	}
237	type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
238	if ((type & 7) != 1) {
239	cpu_abort(CPU(cpu), "invalid tss type");
240	}
241	shift = type >> 3;
242	index = (dpl * 4 + 2) << shift;
243	if (index + (4 << shift) - 1 > env->tr.limit) {
244	raise_exception_err_ra(env, EXCP0A_TSS, env->tr.selector & 0xfffc, retaddr);
245	}
246	if (shift == 0) {
247	*esp_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index, retaddr);
248	*ss_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index + 2, retaddr);
249	} else {
250	*esp_ptr = cpu_ldl_kernel_ra(env, env->tr.base + index, retaddr);
251	*ss_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index + 4, retaddr);
252	}
253	}
254
255	static void tss_load_seg(CPUX86State *env, X86Seg seg_reg, int selector,
256	int cpl, uintptr_t retaddr)
257	{
258	uint32_t e1, e2;
259	int rpl, dpl;
260
261	if ((selector & 0xfffc) != 0) {
262	if (load_segment_ra(env, &e1, &e2, selector, retaddr) != 0) {
263	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
264	}
265	if (!(e2 & DESC_S_MASK)) {
266	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
267	}
268	rpl = selector & 3;
269	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
270	if (seg_reg == R_CS) {
271	if (!(e2 & DESC_CS_MASK)) {
272	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
273	}
274	if (dpl != rpl) {
275	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
276	}
277	} else if (seg_reg == R_SS) {
278	/* SS must be writable data */
279	if ((e2 & DESC_CS_MASK) \|\| !(e2 & DESC_W_MASK)) {
280	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
281	}
282	if (dpl != cpl \|\| dpl != rpl) {
283	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
284	}
285	} else {
286	/* not readable code */
287	if ((e2 & DESC_CS_MASK) && !(e2 & DESC_R_MASK)) {
288	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
289	}
290	/* if data or non conforming code, checks the rights */
291	if (((e2 >> DESC_TYPE_SHIFT) & 0xf) < 12) {
292	if (dpl < cpl \|\| dpl < rpl) {
293	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
294	}
295	}
296	}
297	if (!(e2 & DESC_P_MASK)) {
298	raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, retaddr);
299	}
300	cpu_x86_load_seg_cache(env, seg_reg, selector,
301	get_seg_base(e1, e2),
302	get_seg_limit(e1, e2),
303	e2);
304	} else {
305	if (seg_reg == R_SS \|\| seg_reg == R_CS) {
306	raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
307	}
308	}
309	}
310
311	static void tss_set_busy(CPUX86State *env, int tss_selector, bool value,
312	uintptr_t retaddr)
313	{
314	target_ulong ptr = env->gdt.base + (tss_selector & ~7);
315	uint32_t e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
316
317	if (value) {
318	e2 \|= DESC_TSS_BUSY_MASK;
319	} else {
320	e2 &= ~DESC_TSS_BUSY_MASK;
321	}
322
323	cpu_stl_kernel_ra(env, ptr + 4, e2, retaddr);
324	}
325
326	#define SWITCH_TSS_JMP 0
327	#define SWITCH_TSS_IRET 1
328	#define SWITCH_TSS_CALL 2
329
330	static void switch_tss_ra(CPUX86State *env, int tss_selector,
331	uint32_t e1, uint32_t e2, int source,
332	uint32_t next_eip, bool has_error_code,
333	uint32_t error_code, uintptr_t retaddr)
334	{
335	int tss_limit, tss_limit_max, type, old_tss_limit_max, old_type, i;
336	target_ulong tss_base;
337	uint32_t new_regs[8], new_segs[6];
338	uint32_t new_eflags, new_eip, new_cr3, new_ldt, new_trap;
339	uint32_t old_eflags, eflags_mask;
340	SegmentCache *dt;
341	int mmu_index, index;
342	target_ulong ptr;
343	X86Access old, new;
344
345	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
346	LOG_PCALL("switch_tss: sel=0x%04x type=%d src=%d\n", tss_selector, type,
347	source);
348
349	/* if task gate, we read the TSS segment and we load it */
350	if (type == 5) {
351	if (!(e2 & DESC_P_MASK)) {
352	raise_exception_err_ra(env, EXCP0B_NOSEG, tss_selector & 0xfffc, retaddr);
353	}
354	tss_selector = e1 >> 16;
355	if (tss_selector & 4) {
356	raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
357	}
358	if (load_segment_ra(env, &e1, &e2, tss_selector, retaddr) != 0) {
359	raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
360	}
361	if (e2 & DESC_S_MASK) {
362	raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
363	}
364	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
365	if ((type & 7) != 1) {
366	raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
367	}
368	}
369
370	if (!(e2 & DESC_P_MASK)) {
371	raise_exception_err_ra(env, EXCP0B_NOSEG, tss_selector & 0xfffc, retaddr);
372	}
373
374	if (type & 8) {
375	tss_limit_max = 103;
376	} else {
377	tss_limit_max = 43;
378	}
379	tss_limit = get_seg_limit(e1, e2);
380	tss_base = get_seg_base(e1, e2);
381	if ((tss_selector & 4) != 0 \|\|
382	tss_limit < tss_limit_max) {
383	raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
384	}
385	old_type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
386	if (old_type & 8) {
387	old_tss_limit_max = 103;
388	} else {
389	old_tss_limit_max = 43;
390	}
391
392	/* new TSS must be busy iff the source is an IRET instruction */
393	if (!!(e2 & DESC_TSS_BUSY_MASK) != (source == SWITCH_TSS_IRET)) {
394	raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
395	}
396
397	/* X86Access avoids memory exceptions during the task switch */
398	mmu_index = cpu_mmu_index_kernel(env);
399	access_prepare_mmu(&old, env, env->tr.base, old_tss_limit_max + 1,
400	MMU_DATA_STORE, mmu_index, retaddr);
401
402	if (source == SWITCH_TSS_CALL) {
403	/* Probe for future write of parent task */
404	probe_access(env, tss_base, 2, MMU_DATA_STORE,
405	mmu_index, retaddr);
406	}
407	/* While true tss_limit may be larger, we don't access the iopb here. */
408	access_prepare_mmu(&new, env, tss_base, tss_limit_max + 1,
409	MMU_DATA_LOAD, mmu_index, retaddr);
410
411	/* save the current state in the old TSS */
412	old_eflags = cpu_compute_eflags(env);
413	if (old_type & 8) {
414	/* 32 bit */
415	access_stl(&old, env->tr.base + 0x20, next_eip);
416	access_stl(&old, env->tr.base + 0x24, old_eflags);
417	access_stl(&old, env->tr.base + (0x28 + 0 * 4), env->regs[R_EAX]);
418	access_stl(&old, env->tr.base + (0x28 + 1 * 4), env->regs[R_ECX]);
419	access_stl(&old, env->tr.base + (0x28 + 2 * 4), env->regs[R_EDX]);
420	access_stl(&old, env->tr.base + (0x28 + 3 * 4), env->regs[R_EBX]);
421	access_stl(&old, env->tr.base + (0x28 + 4 * 4), env->regs[R_ESP]);
422	access_stl(&old, env->tr.base + (0x28 + 5 * 4), env->regs[R_EBP]);
423	access_stl(&old, env->tr.base + (0x28 + 6 * 4), env->regs[R_ESI]);
424	access_stl(&old, env->tr.base + (0x28 + 7 * 4), env->regs[R_EDI]);
425	for (i = 0; i < 6; i++) {
426	access_stw(&old, env->tr.base + (0x48 + i * 4),
427	env->segs[i].selector);
428	}
429	} else {
430	/* 16 bit */
431	access_stw(&old, env->tr.base + 0x0e, next_eip);
432	access_stw(&old, env->tr.base + 0x10, old_eflags);
433	access_stw(&old, env->tr.base + (0x12 + 0 * 2), env->regs[R_EAX]);
434	access_stw(&old, env->tr.base + (0x12 + 1 * 2), env->regs[R_ECX]);
435	access_stw(&old, env->tr.base + (0x12 + 2 * 2), env->regs[R_EDX]);
436	access_stw(&old, env->tr.base + (0x12 + 3 * 2), env->regs[R_EBX]);
437	access_stw(&old, env->tr.base + (0x12 + 4 * 2), env->regs[R_ESP]);
438	access_stw(&old, env->tr.base + (0x12 + 5 * 2), env->regs[R_EBP]);
439	access_stw(&old, env->tr.base + (0x12 + 6 * 2), env->regs[R_ESI]);
440	access_stw(&old, env->tr.base + (0x12 + 7 * 2), env->regs[R_EDI]);
441	for (i = 0; i < 4; i++) {
442	access_stw(&old, env->tr.base + (0x22 + i * 2),
443	env->segs[i].selector);
444	}
445	}
446
447	/* read all the registers from the new TSS */
448	if (type & 8) {
449	/* 32 bit */
450	new_cr3 = access_ldl(&new, tss_base + 0x1c);
451	new_eip = access_ldl(&new, tss_base + 0x20);
452	new_eflags = access_ldl(&new, tss_base + 0x24);
453	for (i = 0; i < 8; i++) {
454	new_regs[i] = access_ldl(&new, tss_base + (0x28 + i * 4));
455	}
456	for (i = 0; i < 6; i++) {
457	new_segs[i] = access_ldw(&new, tss_base + (0x48 + i * 4));
458	}
459	new_ldt = access_ldw(&new, tss_base + 0x60);
460	new_trap = access_ldw(&new, tss_base + 0x64) & 1;
461	} else {
462	/* 16 bit */
463	new_cr3 = 0;
464	new_eip = access_ldw(&new, tss_base + 0x0e);
465	new_eflags = access_ldw(&new, tss_base + 0x10);
466	for (i = 0; i < 8; i++) {
467	new_regs[i] = access_ldw(&new, tss_base + (0x12 + i * 2));
468	}
469	for (i = 0; i < 4; i++) {
470	new_segs[i] = access_ldw(&new, tss_base + (0x22 + i * 2));
471	}
472	new_ldt = access_ldw(&new, tss_base + 0x2a);
473	new_segs[R_FS] = 0;
474	new_segs[R_GS] = 0;
475	new_trap = 0;
476	}
477
478	/* clear busy bit (it is restartable) */
479	if (source == SWITCH_TSS_JMP \|\| source == SWITCH_TSS_IRET) {
480	tss_set_busy(env, env->tr.selector, 0, retaddr);
481	}
482
483	if (source == SWITCH_TSS_IRET) {
484	old_eflags &= ~NT_MASK;
485	if (old_type & 8) {
486	access_stl(&old, env->tr.base + 0x24, old_eflags);
487	} else {
488	access_stw(&old, env->tr.base + 0x10, old_eflags);
489	}
490	}
491
492	if (source == SWITCH_TSS_CALL) {
493	/*
494	* Thanks to the probe_access above, we know the first two
495	* bytes addressed by &new are writable too.
496	*/
497	access_stw(&new, tss_base, env->tr.selector);
498	new_eflags \|= NT_MASK;
499	}
500
501	/* set busy bit */
502	if (source == SWITCH_TSS_JMP \|\| source == SWITCH_TSS_CALL) {
503	tss_set_busy(env, tss_selector, 1, retaddr);
504	}
505
506	/* set the new CPU state */
507
508	/* now if an exception occurs, it will occur in the next task context */
509
510	env->cr[0] \|= CR0_TS_MASK;
511	env->hflags \|= HF_TS_MASK;
512	env->tr.selector = tss_selector;
513	env->tr.base = tss_base;
514	env->tr.limit = tss_limit;
515	env->tr.flags = e2 & ~DESC_TSS_BUSY_MASK;
516
517	if ((type & 8) && (env->cr[0] & CR0_PG_MASK)) {
518	cpu_x86_update_cr3(env, new_cr3);
519	}
520
521	/* load all registers without an exception, then reload them with
522	possible exception */
523	env->eip = new_eip;
524	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \|
525	IF_MASK \| IOPL_MASK \| VM_MASK \| RF_MASK \| NT_MASK;
526	if (type & 8) {
527	cpu_load_eflags(env, new_eflags, eflags_mask);
528	for (i = 0; i < 8; i++) {
529	env->regs[i] = new_regs[i];
530	}
531	} else {
532	cpu_load_eflags(env, new_eflags, eflags_mask & 0xffff);
533	for (i = 0; i < 8; i++) {
534	env->regs[i] = (env->regs[i] & 0xffff0000) \| new_regs[i];
535	}
536	}
537	if (new_eflags & VM_MASK) {
538	for (i = 0; i < 6; i++) {
539	load_seg_vm(env, i, new_segs[i]);
540	}
541	} else {
542	/* first just selectors as the rest may trigger exceptions */
543	for (i = 0; i < 6; i++) {
544	cpu_x86_load_seg_cache(env, i, new_segs[i], 0, 0, 0);
545	}
546	}
547
548	env->ldt.selector = new_ldt & ~4;
549	env->ldt.base = 0;
550	env->ldt.limit = 0;
551	env->ldt.flags = 0;
552
553	/* load the LDT */
554	if (new_ldt & 4) {
555	raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
556	}
557
558	if ((new_ldt & 0xfffc) != 0) {
559	dt = &env->gdt;
560	index = new_ldt & ~7;
561	if ((index + 7) > dt->limit) {
562	raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
563	}
564	ptr = dt->base + index;
565	e1 = cpu_ldl_kernel_ra(env, ptr, retaddr);
566	e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
567	if ((e2 & DESC_S_MASK) \|\| ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {
568	raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
569	}
570	if (!(e2 & DESC_P_MASK)) {
571	raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
572	}
573	load_seg_cache_raw_dt(&env->ldt, e1, e2);
574	}
575
576	/* load the segments */
577	if (!(new_eflags & VM_MASK)) {
578	int cpl = new_segs[R_CS] & 3;
579	tss_load_seg(env, R_CS, new_segs[R_CS], cpl, retaddr);
580	tss_load_seg(env, R_SS, new_segs[R_SS], cpl, retaddr);
581	tss_load_seg(env, R_ES, new_segs[R_ES], cpl, retaddr);
582	tss_load_seg(env, R_DS, new_segs[R_DS], cpl, retaddr);
583	tss_load_seg(env, R_FS, new_segs[R_FS], cpl, retaddr);
584	tss_load_seg(env, R_GS, new_segs[R_GS], cpl, retaddr);
585	}
586
587	/* check that env->eip is in the CS segment limits */
588	if (new_eip > env->segs[R_CS].limit) {
589	/* XXX: different exception if CALL? */
590	raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
591	}
592
593	#ifndef CONFIG_USER_ONLY
594	/* reset local breakpoints */
595	if (env->dr[7] & DR7_LOCAL_BP_MASK) {
596	cpu_x86_update_dr7(env, env->dr[7] & ~DR7_LOCAL_BP_MASK);
597	}
598	#endif
599
600	if (has_error_code) {
601	int cpl = env->hflags & HF_CPL_MASK;
602	StackAccess sa;
603
604	/* push the error code */
605	sa.env = env;
606	sa.ra = retaddr;
607	sa.mmu_index = x86_mmu_index_pl(env, cpl);
608	sa.sp = env->regs[R_ESP];
609	if (env->segs[R_SS].flags & DESC_B_MASK) {
610	sa.sp_mask = 0xffffffff;
611	} else {
612	sa.sp_mask = 0xffff;
613	}
614	sa.ss_base = env->segs[R_SS].base;
615	if (type & 8) {
616	pushl(&sa, error_code);
617	} else {
618	pushw(&sa, error_code);
619	}
620	SET_ESP(sa.sp, sa.sp_mask);
621	}
622
623	if (new_trap) {
624	env->dr[6] \|= DR6_BT;
625	raise_exception_ra(env, EXCP01_DB, retaddr);
626	}
627	}
628
629	static void switch_tss(CPUX86State *env, int tss_selector,
630	uint32_t e1, uint32_t e2, int source,
631	uint32_t next_eip, bool has_error_code,
632	int error_code)
633	{
634	switch_tss_ra(env, tss_selector, e1, e2, source, next_eip,
635	has_error_code, error_code, 0);
636	}
637
638	static inline unsigned int get_sp_mask(unsigned int e2)
639	{
640	#ifdef TARGET_X86_64
641	if (e2 & DESC_L_MASK) {
642	return 0;
643	} else
644	#endif
645	if (e2 & DESC_B_MASK) {
646	return 0xffffffff;
647	} else {
648	return 0xffff;
649	}
650	}
651
652	static int exception_is_fault(int intno)
653	{
654	switch (intno) {
655	/*
656	* #DB can be both fault- and trap-like, but it never sets RF=1
657	* in the RFLAGS value pushed on the stack.
658	*/
659	case EXCP01_DB:
660	case EXCP03_INT3:
661	case EXCP04_INTO:
662	case EXCP08_DBLE:
663	case EXCP12_MCHK:
664	return 0;
665	}
666	/* Everything else including reserved exception is a fault. */
667	return 1;
668	}
669
670	int exception_has_error_code(int intno)
671	{
672	switch (intno) {
673	case 8:
674	case 10:
675	case 11:
676	case 12:
677	case 13:
678	case 14:
679	case 17:
680	return 1;
681	}
682	return 0;
683	}
684
685	/* protected mode interrupt */
686	static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,
687	int error_code, unsigned int next_eip,
688	int is_hw)
689	{
690	SegmentCache *dt;
691	target_ulong ptr;
692	int type, dpl, selector, ss_dpl, cpl;
693	int has_error_code, new_stack, shift;
694	uint32_t e1, e2, offset, ss = 0, ss_e1 = 0, ss_e2 = 0;
695	uint32_t old_eip, eflags;
696	int vm86 = env->eflags & VM_MASK;
697	StackAccess sa;
698	bool set_rf;
699
700	has_error_code = 0;
701	if (!is_int && !is_hw) {
702	has_error_code = exception_has_error_code(intno);
703	}
704	if (is_int) {
705	old_eip = next_eip;
706	set_rf = false;
707	} else {
708	old_eip = env->eip;
709	set_rf = exception_is_fault(intno);
710	}
711
712	dt = &env->idt;
713	if (intno * 8 + 7 > dt->limit) {
714	raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);
715	}
716	ptr = dt->base + intno * 8;
717	e1 = cpu_ldl_kernel(env, ptr);
718	e2 = cpu_ldl_kernel(env, ptr + 4);
719	/* check gate type */
720	type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
721	switch (type) {
722	case 5: /* task gate */
723	case 6: /* 286 interrupt gate */
724	case 7: /* 286 trap gate */
725	case 14: /* 386 interrupt gate */
726	case 15: /* 386 trap gate */
727	break;
728	default:
729	raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);
730	break;
731	}
732	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
733	cpl = env->hflags & HF_CPL_MASK;
734	/* check privilege if software int */
735	if (is_int && dpl < cpl) {
736	raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);
737	}
738
739	sa.env = env;
740	sa.ra = 0;
741
742	if (type == 5) {
743	/* task gate */
744	/* must do that check here to return the correct error code */
745	if (!(e2 & DESC_P_MASK)) {
746	raise_exception_err(env, EXCP0B_NOSEG, intno * 8 + 2);
747	}
748	switch_tss(env, intno * 8, e1, e2, SWITCH_TSS_CALL, old_eip,
749	has_error_code, error_code);
750	return;
751	}
752
753	/* Otherwise, trap or interrupt gate */
754
755	/* check valid bit */
756	if (!(e2 & DESC_P_MASK)) {
757	raise_exception_err(env, EXCP0B_NOSEG, intno * 8 + 2);
758	}
759	selector = e1 >> 16;
760	offset = (e2 & 0xffff0000) \| (e1 & 0x0000ffff);
761	if ((selector & 0xfffc) == 0) {
762	raise_exception_err(env, EXCP0D_GPF, 0);
763	}
764	if (load_segment(env, &e1, &e2, selector) != 0) {
765	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
766	}
767	if (!(e2 & DESC_S_MASK) \|\| !(e2 & (DESC_CS_MASK))) {
768	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
769	}
770	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
771	if (dpl > cpl) {
772	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
773	}
774	if (!(e2 & DESC_P_MASK)) {
775	raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
776	}
777	if (e2 & DESC_C_MASK) {
778	dpl = cpl;
779	}
780	sa.mmu_index = x86_mmu_index_pl(env, dpl);
781	if (dpl < cpl) {
782	/* to inner privilege */
783	uint32_t esp;
784	get_ss_esp_from_tss(env, &ss, &esp, dpl, 0);
785	if ((ss & 0xfffc) == 0) {
786	raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
787	}
788	if ((ss & 3) != dpl) {
789	raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
790	}
791	if (load_segment(env, &ss_e1, &ss_e2, ss) != 0) {
792	raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
793	}
794	ss_dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
795	if (ss_dpl != dpl) {
796	raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
797	}
798	if (!(ss_e2 & DESC_S_MASK) \|\|
799	(ss_e2 & DESC_CS_MASK) \|\|
800	!(ss_e2 & DESC_W_MASK)) {
801	raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
802	}
803	if (!(ss_e2 & DESC_P_MASK)) {
804	raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
805	}
806	new_stack = 1;
807	sa.sp = esp;
808	sa.sp_mask = get_sp_mask(ss_e2);
809	sa.ss_base = get_seg_base(ss_e1, ss_e2);
810	} else {
811	/* to same privilege */
812	if (vm86) {
813	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
814	}
815	new_stack = 0;
816	sa.sp = env->regs[R_ESP];
817	sa.sp_mask = get_sp_mask(env->segs[R_SS].flags);
818	sa.ss_base = env->segs[R_SS].base;
819	}
820
821	shift = type >> 3;
822
823	#if 0
824	/* XXX: check that enough room is available */
825	push_size = 6 + (new_stack << 2) + (has_error_code << 1);
826	if (vm86) {
827	push_size += 8;
828	}
829	push_size <<= shift;
830	#endif
831	eflags = cpu_compute_eflags(env);
832	/*
833	* AMD states that code breakpoint #DBs clear RF=0, Intel leaves it
834	* as is. AMD behavior could be implemented in check_hw_breakpoints().
835	*/
836	if (set_rf) {
837	eflags \|= RF_MASK;
838	}
839
840	if (shift == 1) {
841	if (new_stack) {
842	if (vm86) {
843	pushl(&sa, env->segs[R_GS].selector);
844	pushl(&sa, env->segs[R_FS].selector);
845	pushl(&sa, env->segs[R_DS].selector);
846	pushl(&sa, env->segs[R_ES].selector);
847	}
848	pushl(&sa, env->segs[R_SS].selector);
849	pushl(&sa, env->regs[R_ESP]);
850	}
851	pushl(&sa, eflags);
852	pushl(&sa, env->segs[R_CS].selector);
853	pushl(&sa, old_eip);
854	if (has_error_code) {
855	pushl(&sa, error_code);
856	}
857	} else {
858	if (new_stack) {
859	if (vm86) {
860	pushw(&sa, env->segs[R_GS].selector);
861	pushw(&sa, env->segs[R_FS].selector);
862	pushw(&sa, env->segs[R_DS].selector);
863	pushw(&sa, env->segs[R_ES].selector);
864	}
865	pushw(&sa, env->segs[R_SS].selector);
866	pushw(&sa, env->regs[R_ESP]);
867	}
868	pushw(&sa, eflags);
869	pushw(&sa, env->segs[R_CS].selector);
870	pushw(&sa, old_eip);
871	if (has_error_code) {
872	pushw(&sa, error_code);
873	}
874	}
875
876	/* interrupt gate clear IF mask */
877	if ((type & 1) == 0) {
878	env->eflags &= ~IF_MASK;
879	}
880	env->eflags &= ~(TF_MASK \| VM_MASK \| RF_MASK \| NT_MASK);
881
882	if (new_stack) {
883	if (vm86) {
884	cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0, 0);
885	cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0, 0);
886	cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0, 0);
887	cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0, 0);
888	}
889	ss = (ss & ~3) \| dpl;
890	cpu_x86_load_seg_cache(env, R_SS, ss, sa.ss_base,
891	get_seg_limit(ss_e1, ss_e2), ss_e2);
892	}
893	SET_ESP(sa.sp, sa.sp_mask);
894
895	selector = (selector & ~3) \| dpl;
896	cpu_x86_load_seg_cache(env, R_CS, selector,
897	get_seg_base(e1, e2),
898	get_seg_limit(e1, e2),
899	e2);
900	env->eip = offset;
901	}
902
903	#ifdef TARGET_X86_64
904
905	static void pushq(StackAccess *sa, uint64_t val)
906	{
907	sa->sp -= 8;
908	cpu_stq_le_mmuidx_ra(sa->env, sa->sp, val, sa->mmu_index, sa->ra);
909	}
910
911	static uint64_t popq(StackAccess *sa)
912	{
913	uint64_t ret = cpu_ldq_le_mmuidx_ra(sa->env, sa->sp, sa->mmu_index, sa->ra);
914	sa->sp += 8;
915	return ret;
916	}
917
918	static inline target_ulong get_rsp_from_tss(CPUX86State *env, int level)
919	{
920	X86CPU *cpu = env_archcpu(env);
921	int index, pg_mode;
922	target_ulong rsp;
923	int32_t sext;
924
925	#if 0
926	printf("TR: base=" TARGET_FMT_lx " limit=%x\n",
927	env->tr.base, env->tr.limit);
928	#endif
929
930	if (!(env->tr.flags & DESC_P_MASK)) {
931	cpu_abort(CPU(cpu), "invalid tss");
932	}
933	index = 8 * level + 4;
934	if ((index + 7) > env->tr.limit) {
935	raise_exception_err(env, EXCP0A_TSS, env->tr.selector & 0xfffc);
936	}
937
938	rsp = cpu_ldq_kernel(env, env->tr.base + index);
939
940	/* test virtual address sign extension */
941	pg_mode = get_pg_mode(env);
942	sext = (int64_t)rsp >> (pg_mode & PG_MODE_LA57 ? 56 : 47);
943	if (sext != 0 && sext != -1) {
944	raise_exception_err(env, EXCP0C_STACK, 0);
945	}
946
947	return rsp;
948	}
949
950	/* 64 bit interrupt */
951	static void do_interrupt64(CPUX86State *env, int intno, int is_int,
952	int error_code, target_ulong next_eip, int is_hw)
953	{
954	SegmentCache *dt;
955	target_ulong ptr;
956	int type, dpl, selector, cpl, ist;
957	int has_error_code, new_stack;
958	uint32_t e1, e2, e3, eflags;
959	target_ulong old_eip, offset;
960	bool set_rf;
961	StackAccess sa;
962
963	has_error_code = 0;
964	if (!is_int && !is_hw) {
965	has_error_code = exception_has_error_code(intno);
966	}
967	if (is_int) {
968	old_eip = next_eip;
969	set_rf = false;
970	} else {
971	old_eip = env->eip;
972	set_rf = exception_is_fault(intno);
973	}
974
975	dt = &env->idt;
976	if (intno * 16 + 15 > dt->limit) {
977	raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);
978	}
979	ptr = dt->base + intno * 16;
980	e1 = cpu_ldl_kernel(env, ptr);
981	e2 = cpu_ldl_kernel(env, ptr + 4);
982	e3 = cpu_ldl_kernel(env, ptr + 8);
983	/* check gate type */
984	type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
985	switch (type) {
986	case 14: /* 386 interrupt gate */
987	case 15: /* 386 trap gate */
988	break;
989	default:
990	raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);
991	break;
992	}
993	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
994	cpl = env->hflags & HF_CPL_MASK;
995	/* check privilege if software int */
996	if (is_int && dpl < cpl) {
997	raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);
998	}
999	/* check valid bit */
1000	if (!(e2 & DESC_P_MASK)) {
1001	raise_exception_err(env, EXCP0B_NOSEG, intno * 8 + 2);
1002	}
1003	selector = e1 >> 16;
1004	offset = ((target_ulong)e3 << 32) \| (e2 & 0xffff0000) \| (e1 & 0x0000ffff);
1005	ist = e2 & 7;
1006	if ((selector & 0xfffc) == 0) {
1007	raise_exception_err(env, EXCP0D_GPF, 0);
1008	}
1009
1010	if (load_segment(env, &e1, &e2, selector) != 0) {
1011	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
1012	}
1013	if (!(e2 & DESC_S_MASK) \|\| !(e2 & (DESC_CS_MASK))) {
1014	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
1015	}
1016	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1017	if (dpl > cpl) {
1018	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
1019	}
1020	if (!(e2 & DESC_P_MASK)) {
1021	raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
1022	}
1023	if (!(e2 & DESC_L_MASK) \|\| (e2 & DESC_B_MASK)) {
1024	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
1025	}
1026	if (e2 & DESC_C_MASK) {
1027	dpl = cpl;
1028	}
1029
1030	sa.env = env;
1031	sa.ra = 0;
1032	sa.mmu_index = x86_mmu_index_pl(env, dpl);
1033	sa.sp_mask = -1;
1034	sa.ss_base = 0;
1035	if (dpl < cpl \|\| ist != 0) {
1036	/* to inner privilege */
1037	new_stack = 1;
1038	sa.sp = get_rsp_from_tss(env, ist != 0 ? ist + 3 : dpl);
1039	} else {
1040	/* to same privilege */
1041	if (env->eflags & VM_MASK) {
1042	raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
1043	}
1044	new_stack = 0;
1045	sa.sp = env->regs[R_ESP];
1046	}
1047	sa.sp &= ~0xfLL; /* align stack */
1048
1049	/* See do_interrupt_protected. */
1050	eflags = cpu_compute_eflags(env);
1051	if (set_rf) {
1052	eflags \|= RF_MASK;
1053	}
1054
1055	pushq(&sa, env->segs[R_SS].selector);
1056	pushq(&sa, env->regs[R_ESP]);
1057	pushq(&sa, eflags);
1058	pushq(&sa, env->segs[R_CS].selector);
1059	pushq(&sa, old_eip);
1060	if (has_error_code) {
1061	pushq(&sa, error_code);
1062	}
1063
1064	/* interrupt gate clear IF mask */
1065	if ((type & 1) == 0) {
1066	env->eflags &= ~IF_MASK;
1067	}
1068	env->eflags &= ~(TF_MASK \| VM_MASK \| RF_MASK \| NT_MASK);
1069
1070	if (new_stack) {
1071	uint32_t ss = 0 \| dpl; /* SS = NULL selector with RPL = new CPL */
1072	cpu_x86_load_seg_cache(env, R_SS, ss, 0, 0, dpl << DESC_DPL_SHIFT);
1073	}
1074	env->regs[R_ESP] = sa.sp;
1075
1076	selector = (selector & ~3) \| dpl;
1077	cpu_x86_load_seg_cache(env, R_CS, selector,
1078	get_seg_base(e1, e2),
1079	get_seg_limit(e1, e2),
1080	e2);
1081	env->eip = offset;
1082	}
1083	#endif /* TARGET_X86_64 */
1084
1085	void helper_sysret(CPUX86State *env, int dflag)
1086	{
1087	int cpl, selector;
1088
1089	if (!(env->efer & MSR_EFER_SCE)) {
1090	raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
1091	}
1092	cpl = env->hflags & HF_CPL_MASK;
1093	if (!(env->cr[0] & CR0_PE_MASK) \|\| cpl != 0) {
1094	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
1095	}
1096	selector = (env->star >> 48) & 0xffff;
1097	#ifdef TARGET_X86_64
1098	if (env->hflags & HF_LMA_MASK) {
1099	if (dflag == 2) {
1100	uint64_t new_rip = env->regs[R_ECX];
1101	if (IS_INTEL_CPU(env)) {
1102	int shift = (get_pg_mode(env) & PG_MODE_LA57) ? 56 : 47;
1103	int64_t sext = (int64_t)new_rip >> shift;
1104	if (sext != 0 && sext != -1) {
1105	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
1106	}
1107	}
1108	cpu_x86_load_seg_cache(env, R_CS, (selector + 16) \| 3,
1109	0, 0xffffffff,
1110	DESC_G_MASK \| DESC_P_MASK \|
1111	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1112	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK \|
1113	DESC_L_MASK);
1114	env->eip = new_rip;
1115	} else {
1116	cpu_x86_load_seg_cache(env, R_CS, selector \| 3,
1117	0, 0xffffffff,
1118	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1119	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1120	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
1121	env->eip = (uint32_t)env->regs[R_ECX];
1122	}
1123	cpu_x86_load_seg_cache(env, R_SS, (selector + 8) \| 3,
1124	0, 0xffffffff,
1125	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1126	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1127	DESC_W_MASK \| DESC_A_MASK);
1128
1129	cpu_load_eflags(env, (uint32_t)(env->regs[11]), TF_MASK \| AC_MASK
1130	\| ID_MASK \| IF_MASK \| IOPL_MASK \| VM_MASK \| RF_MASK \|
1131	NT_MASK);
1132	} else
1133	#endif
1134	{
1135	env->eflags \|= IF_MASK;
1136	cpu_x86_load_seg_cache(env, R_CS, selector \| 3,
1137	0, 0xffffffff,
1138	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1139	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1140	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
1141	env->eip = (uint32_t)env->regs[R_ECX];
1142	cpu_x86_load_seg_cache(env, R_SS, (selector + 8) \| 3,
1143	0, 0xffffffff,
1144	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1145	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1146	DESC_W_MASK \| DESC_A_MASK);
1147	}
1148	}
1149
1150	/* real mode interrupt */
1151	static void do_interrupt_real(CPUX86State *env, int intno, int is_int,
1152	int error_code, unsigned int next_eip)
1153	{
1154	SegmentCache *dt;
1155	target_ulong ptr;
1156	int selector;
1157	uint32_t offset;
1158	uint32_t old_cs, old_eip;
1159	StackAccess sa;
1160
1161	/* real mode (simpler!) */
1162	dt = &env->idt;
1163	if (intno * 4 + 3 > dt->limit) {
1164	raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);
1165	}
1166	ptr = dt->base + intno * 4;
1167	offset = cpu_lduw_kernel(env, ptr);
1168	selector = cpu_lduw_kernel(env, ptr + 2);
1169
1170	sa.env = env;
1171	sa.ra = 0;
1172	sa.sp = env->regs[R_ESP];
1173	sa.sp_mask = get_sp_mask(env->segs[R_SS].flags);
1174	sa.ss_base = env->segs[R_SS].base;
1175	sa.mmu_index = x86_mmu_index_pl(env, 0);
1176
1177	if (is_int) {
1178	old_eip = next_eip;
1179	} else {
1180	old_eip = env->eip;
1181	}
1182	old_cs = env->segs[R_CS].selector;
1183	/* XXX: use SS segment size? */
1184	pushw(&sa, cpu_compute_eflags(env));
1185	pushw(&sa, old_cs);
1186	pushw(&sa, old_eip);
1187
1188	/* update processor state */
1189	SET_ESP(sa.sp, sa.sp_mask);
1190	env->eip = offset;
1191	env->segs[R_CS].selector = selector;
1192	env->segs[R_CS].base = (selector << 4);
1193	env->eflags &= ~(IF_MASK \| TF_MASK \| AC_MASK \| RF_MASK);
1194	}
1195
1196	/*
1197	* Begin execution of an interruption. is_int is TRUE if coming from
1198	* the int instruction. next_eip is the env->eip value AFTER the interrupt
1199	* instruction. It is only relevant if is_int is TRUE.
1200	*/
1201	void do_interrupt_all(X86CPU *cpu, int intno, int is_int,
1202	int error_code, target_ulong next_eip, int is_hw)
1203	{
1204	CPUX86State *env = &cpu->env;
1205	uint64_t last_pc = env->eip + env->segs[R_CS].base;
1206
1207	if (qemu_loglevel_mask(CPU_LOG_INT)) {
1208	if ((env->cr[0] & CR0_PE_MASK)) {
1209	static int count;
1210
1211	qemu_log("%6d: v=%02x e=%04x i=%d cpl=%d IP=%04x:" TARGET_FMT_lx
1212	" pc=" TARGET_FMT_lx " SP=%04x:" TARGET_FMT_lx,
1213	count, intno, error_code, is_int,
1214	env->hflags & HF_CPL_MASK,
1215	env->segs[R_CS].selector, env->eip,
1216	(int)env->segs[R_CS].base + env->eip,
1217	env->segs[R_SS].selector, env->regs[R_ESP]);
1218	if (intno == 0x0e) {
1219	qemu_log(" CR2=" TARGET_FMT_lx, env->cr[2]);
1220	} else {
1221	qemu_log(" env->regs[R_EAX]=" TARGET_FMT_lx, env->regs[R_EAX]);
1222	}
1223	qemu_log("\n");
1224	log_cpu_state(CPU(cpu), CPU_DUMP_CCOP);
1225	#if 0
1226	{
1227	int i;
1228	target_ulong ptr;
1229
1230	qemu_log(" code=");
1231	ptr = env->segs[R_CS].base + env->eip;
1232	for (i = 0; i < 16; i++) {
1233	qemu_log(" %02x", ldub(ptr + i));
1234	}
1235	qemu_log("\n");
1236	}
1237	#endif
1238	count++;
1239	}
1240	}
1241	if (env->cr[0] & CR0_PE_MASK) {
1242	#if !defined(CONFIG_USER_ONLY)
1243	if (env->hflags & HF_GUEST_MASK) {
1244	handle_even_inj(env, intno, is_int, error_code, is_hw, 0);
1245	}
1246	#endif
1247	#ifdef TARGET_X86_64
1248	if (env->hflags & HF_LMA_MASK) {
1249	do_interrupt64(env, intno, is_int, error_code, next_eip, is_hw);
1250	} else
1251	#endif
1252	{
1253	do_interrupt_protected(env, intno, is_int, error_code, next_eip,
1254	is_hw);
1255	}
1256	} else {
1257	#if !defined(CONFIG_USER_ONLY)
1258	if (env->hflags & HF_GUEST_MASK) {
1259	handle_even_inj(env, intno, is_int, error_code, is_hw, 1);
1260	}
1261	#endif
1262	do_interrupt_real(env, intno, is_int, error_code, next_eip);
1263	}
1264
1265	#if !defined(CONFIG_USER_ONLY)
1266	if (env->hflags & HF_GUEST_MASK) {
1267	CPUState *cs = CPU(cpu);
1268	uint32_t event_inj = x86_ldl_phys(cs, env->vm_vmcb +
1269	offsetof(struct vmcb,
1270	control.event_inj));
1271
1272	x86_stl_phys(cs,
1273	env->vm_vmcb + offsetof(struct vmcb, control.event_inj),
1274	event_inj & ~SVM_EVTINJ_VALID);
1275	}
1276	#endif
1277
1278	qemu_plugin_vcpu_interrupt_cb(CPU(cpu), last_pc);
1279	}
1280
1281	void do_interrupt_x86_hardirq(CPUX86State *env, int intno, int is_hw)
1282	{
1283	do_interrupt_all(env_archcpu(env), intno, 0, 0, 0, is_hw);
1284	}
1285
1286	void helper_lldt(CPUX86State *env, int selector)
1287	{
1288	SegmentCache *dt;
1289	uint32_t e1, e2;
1290	int index, entry_limit;
1291	target_ulong ptr;
1292
1293	selector &= 0xffff;
1294	if ((selector & 0xfffc) == 0) {
1295	/* XXX: NULL selector case: invalid LDT */
1296	env->ldt.base = 0;
1297	env->ldt.limit = 0;
1298	} else {
1299	if (selector & 0x4) {
1300	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1301	}
1302	dt = &env->gdt;
1303	index = selector & ~7;
1304	#ifdef TARGET_X86_64
1305	if (env->hflags & HF_LMA_MASK) {
1306	entry_limit = 15;
1307	} else
1308	#endif
1309	{
1310	entry_limit = 7;
1311	}
1312	if ((index + entry_limit) > dt->limit) {
1313	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1314	}
1315	ptr = dt->base + index;
1316	e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
1317	e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
1318	if ((e2 & DESC_S_MASK) \|\| ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {
1319	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1320	}
1321	if (!(e2 & DESC_P_MASK)) {
1322	raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
1323	}
1324	#ifdef TARGET_X86_64
1325	if (env->hflags & HF_LMA_MASK) {
1326	uint32_t e3;
1327
1328	e3 = cpu_ldl_kernel_ra(env, ptr + 8, GETPC());
1329	load_seg_cache_raw_dt(&env->ldt, e1, e2);
1330	env->ldt.base \|= (target_ulong)e3 << 32;
1331	} else
1332	#endif
1333	{
1334	load_seg_cache_raw_dt(&env->ldt, e1, e2);
1335	}
1336	}
1337	env->ldt.selector = selector;
1338	}
1339
1340	void helper_ltr(CPUX86State *env, int selector)
1341	{
1342	SegmentCache *dt;
1343	uint32_t e1, e2;
1344	int index, type, entry_limit;
1345	target_ulong ptr;
1346
1347	selector &= 0xffff;
1348	if ((selector & 0xfffc) == 0) {
1349	/* NULL selector case: invalid TR */
1350	env->tr.base = 0;
1351	env->tr.limit = 0;
1352	env->tr.flags = 0;
1353	} else {
1354	if (selector & 0x4) {
1355	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1356	}
1357	dt = &env->gdt;
1358	index = selector & ~7;
1359	#ifdef TARGET_X86_64
1360	if (env->hflags & HF_LMA_MASK) {
1361	entry_limit = 15;
1362	} else
1363	#endif
1364	{
1365	entry_limit = 7;
1366	}
1367	if ((index + entry_limit) > dt->limit) {
1368	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1369	}
1370	ptr = dt->base + index;
1371	e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
1372	e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
1373	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
1374	if ((e2 & DESC_S_MASK) \|\|
1375	(type != 1 && type != 9)) {
1376	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1377	}
1378	if (!(e2 & DESC_P_MASK)) {
1379	raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
1380	}
1381	#ifdef TARGET_X86_64
1382	if (env->hflags & HF_LMA_MASK) {
1383	uint32_t e3, e4;
1384
1385	e3 = cpu_ldl_kernel_ra(env, ptr + 8, GETPC());
1386	e4 = cpu_ldl_kernel_ra(env, ptr + 12, GETPC());
1387	if ((e4 >> DESC_TYPE_SHIFT) & 0xf) {
1388	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1389	}
1390	load_seg_cache_raw_dt(&env->tr, e1, e2);
1391	env->tr.base \|= (target_ulong)e3 << 32;
1392	} else
1393	#endif
1394	{
1395	load_seg_cache_raw_dt(&env->tr, e1, e2);
1396	}
1397	e2 \|= DESC_TSS_BUSY_MASK;
1398	cpu_stl_kernel_ra(env, ptr + 4, e2, GETPC());
1399	}
1400	env->tr.selector = selector;
1401	}
1402
1403	/* only works if protected mode and not VM86. seg_reg must be != R_CS */
1404	void helper_load_seg(CPUX86State *env, int seg_reg, int selector)
1405	{
1406	uint32_t e1, e2;
1407	int cpl, dpl, rpl;
1408	SegmentCache *dt;
1409	int index;
1410	target_ulong ptr;
1411
1412	selector &= 0xffff;
1413	cpl = env->hflags & HF_CPL_MASK;
1414	if ((selector & 0xfffc) == 0) {
1415	/* null selector case */
1416	if (seg_reg == R_SS
1417	#ifdef TARGET_X86_64
1418	&& (!(env->hflags & HF_CS64_MASK) \|\| cpl == 3)
1419	#endif
1420	) {
1421	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
1422	}
1423	cpu_x86_load_seg_cache(env, seg_reg, selector, 0, 0, 0);
1424	} else {
1425
1426	if (selector & 0x4) {
1427	dt = &env->ldt;
1428	} else {
1429	dt = &env->gdt;
1430	}
1431	index = selector & ~7;
1432	if ((index + 7) > dt->limit) {
1433	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1434	}
1435	ptr = dt->base + index;
1436	e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
1437	e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
1438
1439	if (!(e2 & DESC_S_MASK)) {
1440	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1441	}
1442	rpl = selector & 3;
1443	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1444	if (seg_reg == R_SS) {
1445	/* must be writable segment */
1446	if ((e2 & DESC_CS_MASK) \|\| !(e2 & DESC_W_MASK)) {
1447	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1448	}
1449	if (rpl != cpl \|\| dpl != cpl) {
1450	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1451	}
1452	} else {
1453	/* must be readable segment */
1454	if ((e2 & (DESC_CS_MASK \| DESC_R_MASK)) == DESC_CS_MASK) {
1455	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1456	}
1457
1458	if (!(e2 & DESC_CS_MASK) \|\| !(e2 & DESC_C_MASK)) {
1459	/* if not conforming code, test rights */
1460	if (dpl < cpl \|\| dpl < rpl) {
1461	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1462	}
1463	}
1464	}
1465
1466	if (!(e2 & DESC_P_MASK)) {
1467	if (seg_reg == R_SS) {
1468	raise_exception_err_ra(env, EXCP0C_STACK, selector & 0xfffc, GETPC());
1469	} else {
1470	raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
1471	}
1472	}
1473
1474	/* set the access bit if not already set */
1475	if (!(e2 & DESC_A_MASK)) {
1476	e2 \|= DESC_A_MASK;
1477	cpu_stl_kernel_ra(env, ptr + 4, e2, GETPC());
1478	}
1479
1480	cpu_x86_load_seg_cache(env, seg_reg, selector,
1481	get_seg_base(e1, e2),
1482	get_seg_limit(e1, e2),
1483	e2);
1484	#if 0
1485	qemu_log("load_seg: sel=0x%04x base=0x%08lx limit=0x%08lx flags=%08x\n",
1486	selector, (unsigned long)sc->base, sc->limit, sc->flags);
1487	#endif
1488	}
1489	}
1490
1491	/* protected mode jump */
1492	void helper_ljmp_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
1493	target_ulong next_eip)
1494	{
1495	int gate_cs, type;
1496	uint32_t e1, e2, cpl, dpl, rpl, limit;
1497
1498	if ((new_cs & 0xfffc) == 0) {
1499	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
1500	}
1501	if (load_segment_ra(env, &e1, &e2, new_cs, GETPC()) != 0) {
1502	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1503	}
1504	cpl = env->hflags & HF_CPL_MASK;
1505	if (e2 & DESC_S_MASK) {
1506	if (!(e2 & DESC_CS_MASK)) {
1507	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1508	}
1509	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1510	if (e2 & DESC_C_MASK) {
1511	/* conforming code segment */
1512	if (dpl > cpl) {
1513	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1514	}
1515	} else {
1516	/* non conforming code segment */
1517	rpl = new_cs & 3;
1518	if (rpl > cpl) {
1519	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1520	}
1521	if (dpl != cpl) {
1522	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1523	}
1524	}
1525	if (!(e2 & DESC_P_MASK)) {
1526	raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
1527	}
1528	limit = get_seg_limit(e1, e2);
1529	if (new_eip > limit &&
1530	(!(env->hflags & HF_LMA_MASK) \|\| !(e2 & DESC_L_MASK))) {
1531	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
1532	}
1533	cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) \| cpl,
1534	get_seg_base(e1, e2), limit, e2);
1535	env->eip = new_eip;
1536	} else {
1537	/* jump to call or task gate */
1538	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1539	rpl = new_cs & 3;
1540	cpl = env->hflags & HF_CPL_MASK;
1541	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
1542
1543	#ifdef TARGET_X86_64
1544	if (env->efer & MSR_EFER_LMA) {
1545	if (type != 12) {
1546	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1547	}
1548	}
1549	#endif
1550	switch (type) {
1551	case 1: /* 286 TSS */
1552	case 9: /* 386 TSS */
1553	case 5: /* task gate */
1554	if (dpl < cpl \|\| dpl < rpl) {
1555	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1556	}
1557	switch_tss_ra(env, new_cs, e1, e2, SWITCH_TSS_JMP, next_eip,
1558	false, 0, GETPC());
1559	break;
1560	case 4: /* 286 call gate */
1561	case 12: /* 386 call gate */
1562	if ((dpl < cpl) \|\| (dpl < rpl)) {
1563	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1564	}
1565	if (!(e2 & DESC_P_MASK)) {
1566	raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
1567	}
1568	gate_cs = e1 >> 16;
1569	new_eip = (e1 & 0xffff);
1570	if (type == 12) {
1571	new_eip \|= (e2 & 0xffff0000);
1572	}
1573
1574	#ifdef TARGET_X86_64
1575	if (env->efer & MSR_EFER_LMA) {
1576	/* load the upper 8 bytes of the 64-bit call gate */
1577	if (load_segment_ra(env, &e1, &e2, new_cs + 8, GETPC())) {
1578	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc,
1579	GETPC());
1580	}
1581	type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
1582	if (type != 0) {
1583	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc,
1584	GETPC());
1585	}
1586	new_eip \|= ((target_ulong)e1) << 32;
1587	}
1588	#endif
1589
1590	if (load_segment_ra(env, &e1, &e2, gate_cs, GETPC()) != 0) {
1591	raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
1592	}
1593	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1594	/* must be code segment */
1595	if (((e2 & (DESC_S_MASK \| DESC_CS_MASK)) !=
1596	(DESC_S_MASK \| DESC_CS_MASK))) {
1597	raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
1598	}
1599	if (((e2 & DESC_C_MASK) && (dpl > cpl)) \|\|
1600	(!(e2 & DESC_C_MASK) && (dpl != cpl))) {
1601	raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
1602	}
1603	#ifdef TARGET_X86_64
1604	if (env->efer & MSR_EFER_LMA) {
1605	if (!(e2 & DESC_L_MASK)) {
1606	raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
1607	}
1608	if (e2 & DESC_B_MASK) {
1609	raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
1610	}
1611	}
1612	#endif
1613	if (!(e2 & DESC_P_MASK)) {
1614	raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
1615	}
1616	limit = get_seg_limit(e1, e2);
1617	if (new_eip > limit &&
1618	(!(env->hflags & HF_LMA_MASK) \|\| !(e2 & DESC_L_MASK))) {
1619	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
1620	}
1621	cpu_x86_load_seg_cache(env, R_CS, (gate_cs & 0xfffc) \| cpl,
1622	get_seg_base(e1, e2), limit, e2);
1623	env->eip = new_eip;
1624	break;
1625	default:
1626	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1627	break;
1628	}
1629	}
1630	}
1631
1632	/* real mode call */
1633	void helper_lcall_real(CPUX86State *env, uint32_t new_cs, uint32_t new_eip,
1634	int shift, uint32_t next_eip)
1635	{
1636	StackAccess sa;
1637
1638	sa.env = env;
1639	sa.ra = GETPC();
1640	sa.sp = env->regs[R_ESP];
1641	sa.sp_mask = get_sp_mask(env->segs[R_SS].flags);
1642	sa.ss_base = env->segs[R_SS].base;
1643	sa.mmu_index = x86_mmu_index_pl(env, 0);
1644
1645	if (shift) {
1646	pushl(&sa, env->segs[R_CS].selector);
1647	pushl(&sa, next_eip);
1648	} else {
1649	pushw(&sa, env->segs[R_CS].selector);
1650	pushw(&sa, next_eip);
1651	}
1652
1653	SET_ESP(sa.sp, sa.sp_mask);
1654	env->eip = new_eip;
1655	env->segs[R_CS].selector = new_cs;
1656	env->segs[R_CS].base = (new_cs << 4);
1657	}
1658
1659	/* protected mode call */
1660	void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
1661	int shift, target_ulong next_eip)
1662	{
1663	int new_stack, i;
1664	uint32_t e1, e2, cpl, dpl, rpl, selector, param_count;
1665	uint32_t ss = 0, ss_e1 = 0, ss_e2 = 0, type, ss_dpl;
1666	uint32_t val, limit, old_sp_mask;
1667	target_ulong old_ssp, offset;
1668	StackAccess sa;
1669
1670	LOG_PCALL("lcall %04x:" TARGET_FMT_lx " s=%d\n", new_cs, new_eip, shift);
1671	LOG_PCALL_STATE(env_cpu(env));
1672	if ((new_cs & 0xfffc) == 0) {
1673	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
1674	}
1675	if (load_segment_ra(env, &e1, &e2, new_cs, GETPC()) != 0) {
1676	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1677	}
1678	cpl = env->hflags & HF_CPL_MASK;
1679	LOG_PCALL("desc=%08x:%08x\n", e1, e2);
1680
1681	sa.env = env;
1682	sa.ra = GETPC();
1683
1684	if (e2 & DESC_S_MASK) {
1685	/* "normal" far call, no stack switch possible */
1686	if (!(e2 & DESC_CS_MASK)) {
1687	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1688	}
1689	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1690	if (e2 & DESC_C_MASK) {
1691	/* conforming code segment */
1692	if (dpl > cpl) {
1693	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1694	}
1695	} else {
1696	/* non conforming code segment */
1697	rpl = new_cs & 3;
1698	if (rpl > cpl) {
1699	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1700	}
1701	if (dpl != cpl) {
1702	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1703	}
1704	}
1705	if (!(e2 & DESC_P_MASK)) {
1706	raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
1707	}
1708
1709	sa.mmu_index = x86_mmu_index_pl(env, cpl);
1710	#ifdef TARGET_X86_64
1711	/* XXX: check 16/32 bit cases in long mode */
1712	if (shift == 2) {
1713	/* 64 bit case */
1714	sa.sp = env->regs[R_ESP];
1715	sa.sp_mask = -1;
1716	sa.ss_base = 0;
1717	pushq(&sa, env->segs[R_CS].selector);
1718	pushq(&sa, next_eip);
1719	/* from this point, not restartable */
1720	env->regs[R_ESP] = sa.sp;
1721	cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) \| cpl,
1722	get_seg_base(e1, e2),
1723	get_seg_limit(e1, e2), e2);
1724	env->eip = new_eip;
1725	} else
1726	#endif
1727	{
1728	sa.sp = env->regs[R_ESP];
1729	sa.sp_mask = get_sp_mask(env->segs[R_SS].flags);
1730	sa.ss_base = env->segs[R_SS].base;
1731	if (shift) {
1732	pushl(&sa, env->segs[R_CS].selector);
1733	pushl(&sa, next_eip);
1734	} else {
1735	pushw(&sa, env->segs[R_CS].selector);
1736	pushw(&sa, next_eip);
1737	}
1738
1739	limit = get_seg_limit(e1, e2);
1740	if (new_eip > limit) {
1741	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1742	}
1743	/* from this point, not restartable */
1744	SET_ESP(sa.sp, sa.sp_mask);
1745	cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) \| cpl,
1746	get_seg_base(e1, e2), limit, e2);
1747	env->eip = new_eip;
1748	}
1749	} else {
1750	/* check gate type */
1751	type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
1752	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1753	rpl = new_cs & 3;
1754
1755	#ifdef TARGET_X86_64
1756	if (env->efer & MSR_EFER_LMA) {
1757	if (type != 12) {
1758	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1759	}
1760	}
1761	#endif
1762
1763	switch (type) {
1764	case 1: /* available 286 TSS */
1765	case 9: /* available 386 TSS */
1766	case 5: /* task gate */
1767	if (dpl < cpl \|\| dpl < rpl) {
1768	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1769	}
1770	switch_tss_ra(env, new_cs, e1, e2, SWITCH_TSS_CALL, next_eip,
1771	false, 0, GETPC());
1772	return;
1773	case 4: /* 286 call gate */
1774	case 12: /* 386 call gate */
1775	break;
1776	default:
1777	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1778	break;
1779	}
1780	shift = type >> 3;
1781
1782	if (dpl < cpl \|\| dpl < rpl) {
1783	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
1784	}
1785	/* check valid bit */
1786	if (!(e2 & DESC_P_MASK)) {
1787	raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
1788	}
1789	selector = e1 >> 16;
1790	param_count = e2 & 0x1f;
1791	offset = (e2 & 0xffff0000) \| (e1 & 0x0000ffff);
1792	#ifdef TARGET_X86_64
1793	if (env->efer & MSR_EFER_LMA) {
1794	/* load the upper 8 bytes of the 64-bit call gate */
1795	if (load_segment_ra(env, &e1, &e2, new_cs + 8, GETPC())) {
1796	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc,
1797	GETPC());
1798	}
1799	type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
1800	if (type != 0) {
1801	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc,
1802	GETPC());
1803	}
1804	offset \|= ((target_ulong)e1) << 32;
1805	}
1806	#endif
1807	if ((selector & 0xfffc) == 0) {
1808	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
1809	}
1810
1811	if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
1812	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1813	}
1814	if (!(e2 & DESC_S_MASK) \|\| !(e2 & (DESC_CS_MASK))) {
1815	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1816	}
1817	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1818	if (dpl > cpl) {
1819	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1820	}
1821	#ifdef TARGET_X86_64
1822	if (env->efer & MSR_EFER_LMA) {
1823	if (!(e2 & DESC_L_MASK)) {
1824	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1825	}
1826	if (e2 & DESC_B_MASK) {
1827	raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
1828	}
1829	shift++;
1830	}
1831	#endif
1832	if (!(e2 & DESC_P_MASK)) {
1833	raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
1834	}
1835
1836	if (!(e2 & DESC_C_MASK) && dpl < cpl) {
1837	/* to inner privilege */
1838	sa.mmu_index = x86_mmu_index_pl(env, dpl);
1839	#ifdef TARGET_X86_64
1840	if (shift == 2) {
1841	ss = dpl; /* SS = NULL selector with RPL = new CPL */
1842	new_stack = 1;
1843	sa.sp = get_rsp_from_tss(env, dpl);
1844	sa.sp_mask = -1;
1845	sa.ss_base = 0; /* SS base is always zero in IA-32e mode */
1846	LOG_PCALL("new ss:rsp=%04x:%016llx env->regs[R_ESP]="
1847	TARGET_FMT_lx "\n", ss, sa.sp, env->regs[R_ESP]);
1848	} else
1849	#endif
1850	{
1851	uint32_t sp32;
1852	get_ss_esp_from_tss(env, &ss, &sp32, dpl, GETPC());
1853	LOG_PCALL("new ss:esp=%04x:%08x param_count=%d env->regs[R_ESP]="
1854	TARGET_FMT_lx "\n", ss, sp32, param_count,
1855	env->regs[R_ESP]);
1856	if ((ss & 0xfffc) == 0) {
1857	raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
1858	}
1859	if ((ss & 3) != dpl) {
1860	raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
1861	}
1862	if (load_segment_ra(env, &ss_e1, &ss_e2, ss, GETPC()) != 0) {
1863	raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
1864	}
1865	ss_dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
1866	if (ss_dpl != dpl) {
1867	raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
1868	}
1869	if (!(ss_e2 & DESC_S_MASK) \|\|
1870	(ss_e2 & DESC_CS_MASK) \|\|
1871	!(ss_e2 & DESC_W_MASK)) {
1872	raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
1873	}
1874	if (!(ss_e2 & DESC_P_MASK)) {
1875	raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
1876	}
1877
1878	sa.sp = sp32;
1879	sa.sp_mask = get_sp_mask(ss_e2);
1880	sa.ss_base = get_seg_base(ss_e1, ss_e2);
1881	}
1882
1883	/* push_size = ((param_count * 2) + 8) << shift; */
1884	old_sp_mask = get_sp_mask(env->segs[R_SS].flags);
1885	old_ssp = env->segs[R_SS].base;
1886
1887	#ifdef TARGET_X86_64
1888	if (shift == 2) {
1889	/* XXX: verify if new stack address is canonical */
1890	pushq(&sa, env->segs[R_SS].selector);
1891	pushq(&sa, env->regs[R_ESP]);
1892	/* parameters aren't supported for 64-bit call gates */
1893	} else
1894	#endif
1895	if (shift == 1) {
1896	pushl(&sa, env->segs[R_SS].selector);
1897	pushl(&sa, env->regs[R_ESP]);
1898	for (i = param_count - 1; i >= 0; i--) {
1899	val = cpu_ldl_le_data_ra(env,
1900	old_ssp + ((env->regs[R_ESP] + i * 4) & old_sp_mask),
1901	GETPC());
1902	pushl(&sa, val);
1903	}
1904	} else {
1905	pushw(&sa, env->segs[R_SS].selector);
1906	pushw(&sa, env->regs[R_ESP]);
1907	for (i = param_count - 1; i >= 0; i--) {
1908	val = cpu_lduw_le_data_ra(env,
1909	old_ssp + ((env->regs[R_ESP] + i * 2) & old_sp_mask),
1910	GETPC());
1911	pushw(&sa, val);
1912	}
1913	}
1914	new_stack = 1;
1915	} else {
1916	/* to same privilege */
1917	sa.mmu_index = x86_mmu_index_pl(env, cpl);
1918	sa.sp = env->regs[R_ESP];
1919	sa.sp_mask = get_sp_mask(env->segs[R_SS].flags);
1920	sa.ss_base = env->segs[R_SS].base;
1921	/* push_size = (4 << shift); */
1922	new_stack = 0;
1923	}
1924
1925	#ifdef TARGET_X86_64
1926	if (shift == 2) {
1927	pushq(&sa, env->segs[R_CS].selector);
1928	pushq(&sa, next_eip);
1929	} else
1930	#endif
1931	if (shift == 1) {
1932	pushl(&sa, env->segs[R_CS].selector);
1933	pushl(&sa, next_eip);
1934	} else {
1935	pushw(&sa, env->segs[R_CS].selector);
1936	pushw(&sa, next_eip);
1937	}
1938
1939	/* from this point, not restartable */
1940
1941	if (new_stack) {
1942	#ifdef TARGET_X86_64
1943	if (shift == 2) {
1944	cpu_x86_load_seg_cache(env, R_SS, ss, 0, 0, 0);
1945	} else
1946	#endif
1947	{
1948	ss = (ss & ~3) \| dpl;
1949	cpu_x86_load_seg_cache(env, R_SS, ss,
1950	sa.ss_base,
1951	get_seg_limit(ss_e1, ss_e2),
1952	ss_e2);
1953	}
1954	}
1955
1956	selector = (selector & ~3) \| dpl;
1957	cpu_x86_load_seg_cache(env, R_CS, selector,
1958	get_seg_base(e1, e2),
1959	get_seg_limit(e1, e2),
1960	e2);
1961	SET_ESP(sa.sp, sa.sp_mask);
1962	env->eip = offset;
1963	}
1964	}
1965
1966	/* real and vm86 mode iret */
1967	void helper_iret_real(CPUX86State *env, int shift)
1968	{
1969	uint32_t new_cs, new_eip, new_eflags;
1970	int eflags_mask;
1971	StackAccess sa;
1972
1973	sa.env = env;
1974	sa.ra = GETPC();
1975	sa.mmu_index = x86_mmu_index_pl(env, 0);
1976	sa.sp_mask = get_sp_mask(env->segs[R_SS].flags);
1977	sa.sp = env->regs[R_ESP];
1978	sa.ss_base = env->segs[R_SS].base;
1979
1980	if (shift == 1) {
1981	/* 32 bits */
1982	new_eip = popl(&sa);
1983	new_cs = popl(&sa) & 0xffff;
1984	new_eflags = popl(&sa);
1985	} else {
1986	/* 16 bits */
1987	new_eip = popw(&sa);
1988	new_cs = popw(&sa);
1989	new_eflags = popw(&sa);
1990	}
1991	SET_ESP(sa.sp, sa.sp_mask);
1992	env->segs[R_CS].selector = new_cs;
1993	env->segs[R_CS].base = (new_cs << 4);
1994	env->eip = new_eip;
1995	if (env->eflags & VM_MASK) {
1996	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \| IF_MASK \| RF_MASK \|
1997	NT_MASK;
1998	} else {
1999	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \| IF_MASK \| IOPL_MASK \|
2000	RF_MASK \| NT_MASK;
2001	}
2002	if (shift == 0) {
2003	eflags_mask &= 0xffff;
2004	}
2005	cpu_load_eflags(env, new_eflags, eflags_mask);
2006	env->hflags2 &= ~HF2_NMI_MASK;
2007	}
2008
2009	static inline void validate_seg(CPUX86State *env, X86Seg seg_reg, int cpl)
2010	{
2011	int dpl;
2012	uint32_t e2;
2013
2014	/* XXX: on x86_64, we do not want to nullify FS and GS because
2015	they may still contain a valid base. I would be interested to
2016	know how a real x86_64 CPU behaves */
2017	if ((seg_reg == R_FS \|\| seg_reg == R_GS) &&
2018	(env->segs[seg_reg].selector & 0xfffc) == 0) {
2019	return;
2020	}
2021
2022	e2 = env->segs[seg_reg].flags;
2023	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2024	if (!(e2 & DESC_CS_MASK) \|\| !(e2 & DESC_C_MASK)) {
2025	/* data or non conforming code segment */
2026	if (dpl < cpl) {
2027	cpu_x86_load_seg_cache(env, seg_reg, 0,
2028	env->segs[seg_reg].base,
2029	env->segs[seg_reg].limit,
2030	env->segs[seg_reg].flags & ~DESC_P_MASK);
2031	}
2032	}
2033	}
2034
2035	/* protected mode iret */
2036	static inline void helper_ret_protected(CPUX86State *env, int shift,
2037	int is_iret, int addend,
2038	uintptr_t retaddr)
2039	{
2040	uint32_t new_cs, new_eflags, new_ss;
2041	uint32_t new_es, new_ds, new_fs, new_gs;
2042	uint32_t e1, e2, ss_e1, ss_e2;
2043	int cpl, dpl, rpl, eflags_mask, iopl;
2044	target_ulong new_eip, new_esp;
2045	StackAccess sa;
2046
2047	cpl = env->hflags & HF_CPL_MASK;
2048
2049	sa.env = env;
2050	sa.ra = retaddr;
2051	sa.mmu_index = x86_mmu_index_pl(env, cpl);
2052
2053	#ifdef TARGET_X86_64
2054	if (shift == 2) {
2055	sa.sp_mask = -1;
2056	} else
2057	#endif
2058	{
2059	sa.sp_mask = get_sp_mask(env->segs[R_SS].flags);
2060	}
2061	sa.sp = env->regs[R_ESP];
2062	sa.ss_base = env->segs[R_SS].base;
2063	new_eflags = 0; /* avoid warning */
2064	#ifdef TARGET_X86_64
2065	if (shift == 2) {
2066	new_eip = popq(&sa);
2067	new_cs = popq(&sa) & 0xffff;
2068	if (is_iret) {
2069	new_eflags = popq(&sa);
2070	}
2071	} else
2072	#endif
2073	{
2074	if (shift == 1) {
2075	/* 32 bits */
2076	new_eip = popl(&sa);
2077	new_cs = popl(&sa) & 0xffff;
2078	if (is_iret) {
2079	new_eflags = popl(&sa);
2080	bool allow_vm86 = (cpl == 0) && !(env->hflags & HF_LMA_MASK);
2081	if ((new_eflags & VM_MASK) && allow_vm86) {
2082	goto return_to_vm86;
2083	}
2084	}
2085	} else {
2086	/* 16 bits */
2087	new_eip = popw(&sa);
2088	new_cs = popw(&sa);
2089	if (is_iret) {
2090	new_eflags = popw(&sa);
2091	}
2092	}
2093	}
2094	LOG_PCALL("lret new %04x:" TARGET_FMT_lx " s=%d addend=0x%x\n",
2095	new_cs, new_eip, shift, addend);
2096	LOG_PCALL_STATE(env_cpu(env));
2097	if ((new_cs & 0xfffc) == 0) {
2098	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
2099	}
2100	if (load_segment_ra(env, &e1, &e2, new_cs, retaddr) != 0) {
2101	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
2102	}
2103	if (!(e2 & DESC_S_MASK) \|\|
2104	!(e2 & DESC_CS_MASK)) {
2105	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
2106	}
2107	rpl = new_cs & 3;
2108	if (rpl < cpl) {
2109	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
2110	}
2111	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2112	if (e2 & DESC_C_MASK) {
2113	if (dpl > rpl) {
2114	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
2115	}
2116	} else {
2117	if (dpl != rpl) {
2118	raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
2119	}
2120	}
2121	if (!(e2 & DESC_P_MASK)) {
2122	raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, retaddr);
2123	}
2124
2125	sa.sp += addend;
2126	if (rpl == cpl && (!(env->hflags & HF_CS64_MASK) \|\|
2127	((env->hflags & HF_CS64_MASK) && !is_iret))) {
2128	/* return to same privilege level */
2129	cpu_x86_load_seg_cache(env, R_CS, new_cs,
2130	get_seg_base(e1, e2),
2131	get_seg_limit(e1, e2),
2132	e2);
2133	} else {
2134	/* return to different privilege level */
2135	#ifdef TARGET_X86_64
2136	if (shift == 2) {
2137	new_esp = popq(&sa);
2138	new_ss = popq(&sa) & 0xffff;
2139	} else
2140	#endif
2141	{
2142	if (shift == 1) {
2143	/* 32 bits */
2144	new_esp = popl(&sa);
2145	new_ss = popl(&sa) & 0xffff;
2146	} else {
2147	/* 16 bits */
2148	new_esp = popw(&sa);
2149	new_ss = popw(&sa);
2150	}
2151	}
2152	LOG_PCALL("new ss:esp=%04x:" TARGET_FMT_lx "\n",
2153	new_ss, new_esp);
2154	if ((new_ss & 0xfffc) == 0) {
2155	#ifdef TARGET_X86_64
2156	/* NULL ss is allowed in long mode if cpl != 3 */
2157	/* XXX: test CS64? */
2158	if ((env->hflags & HF_LMA_MASK) && rpl != 3) {
2159	cpu_x86_load_seg_cache(env, R_SS, new_ss,
2160	0, 0xffffffff,
2161	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
2162	DESC_S_MASK \| (rpl << DESC_DPL_SHIFT) \|
2163	DESC_W_MASK \| DESC_A_MASK);
2164	ss_e2 = DESC_B_MASK; /* XXX: should not be needed? */
2165	} else
2166	#endif
2167	{
2168	raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
2169	}
2170	} else {
2171	if ((new_ss & 3) != rpl) {
2172	raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
2173	}
2174	if (load_segment_ra(env, &ss_e1, &ss_e2, new_ss, retaddr) != 0) {
2175	raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
2176	}
2177	if (!(ss_e2 & DESC_S_MASK) \|\|
2178	(ss_e2 & DESC_CS_MASK) \|\|
2179	!(ss_e2 & DESC_W_MASK)) {
2180	raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
2181	}
2182	dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
2183	if (dpl != rpl) {
2184	raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
2185	}
2186	if (!(ss_e2 & DESC_P_MASK)) {
2187	raise_exception_err_ra(env, EXCP0B_NOSEG, new_ss & 0xfffc, retaddr);
2188	}
2189	cpu_x86_load_seg_cache(env, R_SS, new_ss,
2190	get_seg_base(ss_e1, ss_e2),
2191	get_seg_limit(ss_e1, ss_e2),
2192	ss_e2);
2193	}
2194
2195	cpu_x86_load_seg_cache(env, R_CS, new_cs,
2196	get_seg_base(e1, e2),
2197	get_seg_limit(e1, e2),
2198	e2);
2199	sa.sp = new_esp;
2200	#ifdef TARGET_X86_64
2201	if (env->hflags & HF_CS64_MASK) {
2202	sa.sp_mask = -1;
2203	} else
2204	#endif
2205	{
2206	sa.sp_mask = get_sp_mask(ss_e2);
2207	}
2208
2209	/* validate data segments */
2210	validate_seg(env, R_ES, rpl);
2211	validate_seg(env, R_DS, rpl);
2212	validate_seg(env, R_FS, rpl);
2213	validate_seg(env, R_GS, rpl);
2214
2215	sa.sp += addend;
2216	}
2217	SET_ESP(sa.sp, sa.sp_mask);
2218	env->eip = new_eip;
2219	if (is_iret) {
2220	/* NOTE: 'cpl' is the _old_ CPL */
2221	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \| RF_MASK \| NT_MASK;
2222	if (cpl == 0) {
2223	eflags_mask \|= IOPL_MASK;
2224	}
2225	iopl = (env->eflags >> IOPL_SHIFT) & 3;
2226	if (cpl <= iopl) {
2227	eflags_mask \|= IF_MASK;
2228	}
2229	if (shift == 0) {
2230	eflags_mask &= 0xffff;
2231	}
2232	cpu_load_eflags(env, new_eflags, eflags_mask);
2233	}
2234	return;
2235
2236	return_to_vm86:
2237	new_esp = popl(&sa);
2238	new_ss = popl(&sa);
2239	new_es = popl(&sa);
2240	new_ds = popl(&sa);
2241	new_fs = popl(&sa);
2242	new_gs = popl(&sa);
2243
2244	/* modify processor state */
2245	cpu_load_eflags(env, new_eflags, TF_MASK \| AC_MASK \| ID_MASK \|
2246	IF_MASK \| IOPL_MASK \| VM_MASK \| NT_MASK \| VIF_MASK \|
2247	VIP_MASK);
2248	load_seg_vm(env, R_CS, new_cs & 0xffff);
2249	load_seg_vm(env, R_SS, new_ss & 0xffff);
2250	load_seg_vm(env, R_ES, new_es & 0xffff);
2251	load_seg_vm(env, R_DS, new_ds & 0xffff);
2252	load_seg_vm(env, R_FS, new_fs & 0xffff);
2253	load_seg_vm(env, R_GS, new_gs & 0xffff);
2254
2255	env->eip = new_eip & 0xffff;
2256	env->regs[R_ESP] = new_esp;
2257	}
2258
2259	void helper_iret_protected(CPUX86State *env, int shift, int next_eip)
2260	{
2261	int tss_selector, type;
2262	uint32_t e1, e2;
2263
2264	/* specific case for TSS */
2265	if (env->eflags & NT_MASK) {
2266	#ifdef TARGET_X86_64
2267	if (env->hflags & HF_LMA_MASK) {
2268	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
2269	}
2270	#endif
2271	tss_selector = cpu_lduw_kernel_ra(env, env->tr.base + 0, GETPC());
2272	if (tss_selector & 4) {
2273	raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
2274	}
2275	if (load_segment_ra(env, &e1, &e2, tss_selector, GETPC()) != 0) {
2276	raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
2277	}
2278	type = (e2 >> DESC_TYPE_SHIFT) & 0x17;
2279	/* NOTE: we check both segment and busy TSS */
2280	if (type != 3) {
2281	raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
2282	}
2283	switch_tss_ra(env, tss_selector, e1, e2, SWITCH_TSS_IRET, next_eip,
2284	false, 0, GETPC());
2285	} else {
2286	helper_ret_protected(env, shift, 1, 0, GETPC());
2287	}
2288	env->hflags2 &= ~HF2_NMI_MASK;
2289	}
2290
2291	void helper_lret_protected(CPUX86State *env, int shift, int addend)
2292	{
2293	helper_ret_protected(env, shift, 0, addend, GETPC());
2294	}
2295
2296	void helper_sysenter(CPUX86State *env)
2297	{
2298	if (env->sysenter_cs == 0) {
2299	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
2300	}
2301	env->eflags &= ~(VM_MASK \| IF_MASK \| RF_MASK);
2302
2303	#ifdef TARGET_X86_64
2304	if (env->hflags & HF_LMA_MASK) {
2305	cpu_x86_load_seg_cache(env, R_CS, env->sysenter_cs & 0xfffc,
2306	0, 0xffffffff,
2307	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
2308	DESC_S_MASK \|
2309	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK \|
2310	DESC_L_MASK);
2311	} else
2312	#endif
2313	{
2314	cpu_x86_load_seg_cache(env, R_CS, env->sysenter_cs & 0xfffc,
2315	0, 0xffffffff,
2316	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
2317	DESC_S_MASK \|
2318	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
2319	}
2320	cpu_x86_load_seg_cache(env, R_SS, (env->sysenter_cs + 8) & 0xfffc,
2321	0, 0xffffffff,
2322	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
2323	DESC_S_MASK \|
2324	DESC_W_MASK \| DESC_A_MASK);
2325	env->regs[R_ESP] = env->sysenter_esp;
2326	env->eip = env->sysenter_eip;
2327	}
2328
2329	void helper_sysexit(CPUX86State *env, int dflag)
2330	{
2331	int cpl;
2332
2333	cpl = env->hflags & HF_CPL_MASK;
2334	if (env->sysenter_cs == 0 \|\| cpl != 0) {
2335	raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
2336	}
2337	#ifdef TARGET_X86_64
2338	if (dflag == 2) {
2339	cpu_x86_load_seg_cache(env, R_CS, ((env->sysenter_cs + 32) & 0xfffc) \|
2340	3, 0, 0xffffffff,
2341	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
2342	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
2343	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK \|
2344	DESC_L_MASK);
2345	cpu_x86_load_seg_cache(env, R_SS, ((env->sysenter_cs + 40) & 0xfffc) \|
2346	3, 0, 0xffffffff,
2347	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
2348	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
2349	DESC_W_MASK \| DESC_A_MASK);
2350	} else
2351	#endif
2352	{
2353	cpu_x86_load_seg_cache(env, R_CS, ((env->sysenter_cs + 16) & 0xfffc) \|
2354	3, 0, 0xffffffff,
2355	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
2356	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
2357	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
2358	cpu_x86_load_seg_cache(env, R_SS, ((env->sysenter_cs + 24) & 0xfffc) \|
2359	3, 0, 0xffffffff,
2360	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
2361	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
2362	DESC_W_MASK \| DESC_A_MASK);
2363	}
2364	env->regs[R_ESP] = env->regs[R_ECX];
2365	env->eip = env->regs[R_EDX];
2366	}
2367
2368	target_ulong helper_lsl(CPUX86State *env, target_ulong selector1)
2369	{
2370	unsigned int limit;
2371	uint32_t e1, e2, selector;
2372	int rpl, dpl, cpl, type;
2373
2374	selector = selector1 & 0xffff;
2375	assert(CC_OP == CC_OP_EFLAGS);
2376	if ((selector & 0xfffc) == 0) {
2377	goto fail;
2378	}
2379	if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
2380	goto fail;
2381	}
2382	rpl = selector & 3;
2383	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2384	cpl = env->hflags & HF_CPL_MASK;
2385	if (e2 & DESC_S_MASK) {
2386	if ((e2 & DESC_CS_MASK) && (e2 & DESC_C_MASK)) {
2387	/* conforming */
2388	} else {
2389	if (dpl < cpl \|\| dpl < rpl) {
2390	goto fail;
2391	}
2392	}
2393	} else {
2394	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
2395	switch (type) {
2396	case 1:
2397	case 2:
2398	case 3:
2399	case 9:
2400	case 11:
2401	break;
2402	default:
2403	goto fail;
2404	}
2405	if (dpl < cpl \|\| dpl < rpl) {
2406	fail:
2407	CC_SRC &= ~CC_Z;
2408	return 0;
2409	}
2410	}
2411	limit = get_seg_limit(e1, e2);
2412	CC_SRC \|= CC_Z;
2413	return limit;
2414	}
2415
2416	target_ulong helper_lar(CPUX86State *env, target_ulong selector1)
2417	{
2418	uint32_t e1, e2, selector;
2419	int rpl, dpl, cpl, type;
2420
2421	selector = selector1 & 0xffff;
2422	assert(CC_OP == CC_OP_EFLAGS);
2423	if ((selector & 0xfffc) == 0) {
2424	goto fail;
2425	}
2426	if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
2427	goto fail;
2428	}
2429	rpl = selector & 3;
2430	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2431	cpl = env->hflags & HF_CPL_MASK;
2432	if (e2 & DESC_S_MASK) {
2433	if ((e2 & DESC_CS_MASK) && (e2 & DESC_C_MASK)) {
2434	/* conforming */
2435	} else {
2436	if (dpl < cpl \|\| dpl < rpl) {
2437	goto fail;
2438	}
2439	}
2440	} else {
2441	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
2442	switch (type) {
2443	case 1:
2444	case 2:
2445	case 3:
2446	case 4:
2447	case 5:
2448	case 9:
2449	case 11:
2450	case 12:
2451	break;
2452	default:
2453	goto fail;
2454	}
2455	if (dpl < cpl \|\| dpl < rpl) {
2456	fail:
2457	CC_SRC &= ~CC_Z;
2458	return 0;
2459	}
2460	}
2461	CC_SRC \|= CC_Z;
2462	return e2 & 0x00f0ff00;
2463	}
2464
2465	void helper_verr(CPUX86State *env, target_ulong selector1)
2466	{
2467	uint32_t e1, e2, eflags, selector;
2468	int rpl, dpl, cpl;
2469
2470	selector = selector1 & 0xffff;
2471	eflags = cpu_cc_compute_all(env) \| CC_Z;
2472	if ((selector & 0xfffc) == 0) {
2473	goto fail;
2474	}
2475	if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
2476	goto fail;
2477	}
2478	if (!(e2 & DESC_S_MASK)) {
2479	goto fail;
2480	}
2481	rpl = selector & 3;
2482	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2483	cpl = env->hflags & HF_CPL_MASK;
2484	if (e2 & DESC_CS_MASK) {
2485	if (!(e2 & DESC_R_MASK)) {
2486	goto fail;
2487	}
2488	if (!(e2 & DESC_C_MASK)) {
2489	if (dpl < cpl \|\| dpl < rpl) {
2490	goto fail;
2491	}
2492	}
2493	} else {
2494	if (dpl < cpl \|\| dpl < rpl) {
2495	fail:
2496	eflags &= ~CC_Z;
2497	}
2498	}
2499	CC_SRC = eflags;
2500	CC_OP = CC_OP_EFLAGS;
2501	}
2502
2503	void helper_verw(CPUX86State *env, target_ulong selector1)
2504	{
2505	uint32_t e1, e2, eflags, selector;
2506	int rpl, dpl, cpl;
2507
2508	selector = selector1 & 0xffff;
2509	eflags = cpu_cc_compute_all(env) \| CC_Z;
2510	if ((selector & 0xfffc) == 0) {
2511	goto fail;
2512	}
2513	if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
2514	goto fail;
2515	}
2516	if (!(e2 & DESC_S_MASK)) {
2517	goto fail;
2518	}
2519	rpl = selector & 3;
2520	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2521	cpl = env->hflags & HF_CPL_MASK;
2522	if (e2 & DESC_CS_MASK) {
2523	goto fail;
2524	} else {
2525	if (dpl < cpl \|\| dpl < rpl) {
2526	goto fail;
2527	}
2528	if (!(e2 & DESC_W_MASK)) {
2529	fail:
2530	eflags &= ~CC_Z;
2531	}
2532	}
2533	CC_SRC = eflags;
2534	CC_OP = CC_OP_EFLAGS;
2535	}