IEMAll.cpp@ 40209

Last change on this file since 40209 was 40209, checked in by vboxsync, 13 years ago
IEM: Implemented missing FPU instructions starting with 0xd8 and adjusted fld m32r and fld m64r.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 271.0 KB

Line
1	/* $Id: IEMAll.cpp 40209 2012-02-22 12:14:21Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2011-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/** @page pg_iem IEM - Interpreted Execution Manager
20	*
21	* The interpreted exeuction manager (IEM) is for executing short guest code
22	* sequences that are causing too many exits / virtualization traps. It will
23	* also be used to interpret single instructions, thus replacing the selective
24	* interpreters in EM and IOM.
25	*
26	* Design goals:
27	* - Relatively small footprint, although we favour speed and correctness
28	* over size.
29	* - Reasonably fast.
30	* - Correctly handle lock prefixed instructions.
31	* - Complete instruction set - eventually.
32	* - Refactorable into a recompiler, maybe.
33	* - Replace EMInterpret*.
34	*
35	* Using the existing disassembler has been considered, however this is thought
36	* to conflict with speed as the disassembler chews things a bit too much while
37	* leaving us with a somewhat complicated state to interpret afterwards.
38	*
39	*
40	* The current code is very much work in progress. You've been warned!
41	*
42	*
43	* @section sec_iem_fpu_instr FPU Instructions
44	*
45	* On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
46	* same or equivalent instructions on the host FPU. To make life easy, we also
47	* let the FPU prioritize the unmasked exceptions for us. This however, only
48	* works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
49	* for FPU exception delivery, because with CR0.NE=0 there is a window where we
50	* can trigger spurious FPU exceptions.
51	*
52	* The guest FPU state is not loaded into the host CPU and kept there till we
53	* leave IEM because the calling conventions have declared an all year open
54	* season on much of the FPU state. For instance an innocent looking call to
55	* memcpy might end up using a whole bunch of XMM or MM registers if the
56	* particular implementation finds it worthwhile.
57	*
58	*
59	* @section sec_iem_logging Logging
60	*
61	* The IEM code uses the \"IEM\" log group for the main logging. The different
62	* logging levels/flags are generally used for the following purposes:
63	* - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
64	* - Flow (LogFlow): Additional exception details, basic enter/exit IEM
65	* state info.
66	* - Level 2 (Log2): ?
67	* - Level 3 (Log3): More detailed enter/exit IEM state info.
68	* - Level 4 (Log4): Decoding mnemonics w/ EIP.
69	* - Level 5 (Log5): Decoding details.
70	* - Level 6 (Log6): Enables/disables the lockstep comparison with REM.
71	*
72	*/
73
74	/*******************************************************************************
75	* Header Files *
76	*******************************************************************************/
77	#define LOG_GROUP LOG_GROUP_IEM
78	#include <VBox/vmm/iem.h>
79	#include <VBox/vmm/pgm.h>
80	#include <VBox/vmm/iom.h>
81	#include <VBox/vmm/em.h>
82	#include <VBox/vmm/tm.h>
83	#include <VBox/vmm/dbgf.h>
84	#ifdef IEM_VERIFICATION_MODE
85	# include <VBox/vmm/rem.h>
86	# include <VBox/vmm/mm.h>
87	#endif
88	#include "IEMInternal.h"
89	#include <VBox/vmm/vm.h>
90	#include <VBox/log.h>
91	#include <VBox/err.h>
92	#include <VBox/param.h>
93	#include <iprt/assert.h>
94	#include <iprt/string.h>
95	#include <iprt/x86.h>
96
97
98	/*******************************************************************************
99	* Structures and Typedefs *
100	*******************************************************************************/
101	/** @typedef PFNIEMOP
102	* Pointer to an opcode decoder function.
103	*/
104
105	/** @def FNIEMOP_DEF
106	* Define an opcode decoder function.
107	*
108	* We're using macors for this so that adding and removing parameters as well as
109	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
110	*
111	* @param a_Name The function name.
112	*/
113
114
115	#if defined(__GNUC__) && defined(RT_ARCH_X86)
116	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PIEMCPU pIemCpu);
117	# define FNIEMOP_DEF(a_Name) \
118	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name (PIEMCPU pIemCpu)
119	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
120	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0)
121	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
122	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1)
123
124	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
125	typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PIEMCPU pIemCpu);
126	# define FNIEMOP_DEF(a_Name) \
127	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu) RT_NO_THROW
128	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
129	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0) RT_NO_THROW
130	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
131	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW
132
133	#elif defined(__GNUC__)
134	typedef VBOXSTRICTRC (* PFNIEMOP)(PIEMCPU pIemCpu);
135	# define FNIEMOP_DEF(a_Name) \
136	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu)
137	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
138	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0)
139	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
140	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1)
141
142	#else
143	typedef VBOXSTRICTRC (* PFNIEMOP)(PIEMCPU pIemCpu);
144	# define FNIEMOP_DEF(a_Name) \
145	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu) RT_NO_THROW
146	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
147	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0) RT_NO_THROW
148	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
149	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW
150
151	#endif
152
153
154	/**
155	* Selector descriptor table entry as fetched by iemMemFetchSelDesc.
156	*/
157	typedef union IEMSELDESC
158	{
159	/** The legacy view. */
160	X86DESC Legacy;
161	/** The long mode view. */
162	X86DESC64 Long;
163	} IEMSELDESC;
164	/** Pointer to a selector descriptor table entry. */
165	typedef IEMSELDESC *PIEMSELDESC;
166
167
168	/*******************************************************************************
169	* Defined Constants And Macros *
170	*******************************************************************************/
171	/** @name IEM status codes.
172	*
173	* Not quite sure how this will play out in the end, just aliasing safe status
174	* codes for now.
175	*
176	* @{ */
177	#define VINF_IEM_RAISED_XCPT VINF_EM_RESCHEDULE
178	/** @} */
179
180	/** Temporary hack to disable the double execution. Will be removed in favor
181	* of a dedicated execution mode in EM. */
182	//#define IEM_VERIFICATION_MODE_NO_REM
183
184	/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
185	* due to GCC lacking knowledge about the value range of a switch. */
186	#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
187
188	/**
189	* Call an opcode decoder function.
190	*
191	* We're using macors for this so that adding and removing parameters can be
192	* done as we please. See FNIEMOP_DEF.
193	*/
194	#define FNIEMOP_CALL(a_pfn) (a_pfn)(pIemCpu)
195
196	/**
197	* Call a common opcode decoder function taking one extra argument.
198	*
199	* We're using macors for this so that adding and removing parameters can be
200	* done as we please. See FNIEMOP_DEF_1.
201	*/
202	#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pIemCpu, a0)
203
204	/**
205	* Call a common opcode decoder function taking one extra argument.
206	*
207	* We're using macors for this so that adding and removing parameters can be
208	* done as we please. See FNIEMOP_DEF_1.
209	*/
210	#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pIemCpu, a0, a1)
211
212	/**
213	* Check if we're currently executing in real or virtual 8086 mode.
214	*
215	* @returns @c true if it is, @c false if not.
216	* @param a_pIemCpu The IEM state of the current CPU.
217	*/
218	#define IEM_IS_REAL_OR_V86_MODE(a_pIemCpu) (CPUMIsGuestInRealOrV86ModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
219
220	/**
221	* Check if we're currently executing in long mode.
222	*
223	* @returns @c true if it is, @c false if not.
224	* @param a_pIemCpu The IEM state of the current CPU.
225	*/
226	#define IEM_IS_LONG_MODE(a_pIemCpu) (CPUMIsGuestInLongModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
227
228	/**
229	* Check if we're currently executing in real mode.
230	*
231	* @returns @c true if it is, @c false if not.
232	* @param a_pIemCpu The IEM state of the current CPU.
233	*/
234	#define IEM_IS_REAL_MODE(a_pIemCpu) (CPUMIsGuestInRealModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
235
236	/**
237	* Tests if an AMD CPUID feature (extended) is marked present - ECX.
238	*/
239	#define IEM_IS_AMD_CPUID_FEATURE_PRESENT_ECX(a_fEcx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, 0, (a_fEcx))
240
241	/**
242	* Tests if an AMD CPUID feature (extended) is marked present - EDX.
243	*/
244	#define IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX(a_fEdx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, (a_fEdx), 0)
245
246	/**
247	* Tests if at least on of the specified AMD CPUID features (extended) are
248	* marked present.
249	*/
250	#define IEM_IS_AMD_CPUID_FEATURES_ANY_PRESENT(a_fEdx, a_fEcx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, (a_fEdx), (a_fEcx))
251
252	/**
253	* Checks if a intel CPUID feature is present.
254	*/
255	#define IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX(a_fEdx) \
256	( ((a_fEdx) & (X86_CPUID_FEATURE_EDX_TSC \| 0)) \
257	\|\| iemRegIsIntelCpuIdFeaturePresent(pIemCpu, (a_fEdx), 0) )
258
259	/**
260	* Check if the address is canonical.
261	*/
262	#define IEM_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
263
264
265	/*******************************************************************************
266	* Global Variables *
267	*******************************************************************************/
268	extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
269
270
271	/** Function table for the ADD instruction. */
272	static const IEMOPBINSIZES g_iemAImpl_add =
273	{
274	iemAImpl_add_u8, iemAImpl_add_u8_locked,
275	iemAImpl_add_u16, iemAImpl_add_u16_locked,
276	iemAImpl_add_u32, iemAImpl_add_u32_locked,
277	iemAImpl_add_u64, iemAImpl_add_u64_locked
278	};
279
280	/** Function table for the ADC instruction. */
281	static const IEMOPBINSIZES g_iemAImpl_adc =
282	{
283	iemAImpl_adc_u8, iemAImpl_adc_u8_locked,
284	iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
285	iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
286	iemAImpl_adc_u64, iemAImpl_adc_u64_locked
287	};
288
289	/** Function table for the SUB instruction. */
290	static const IEMOPBINSIZES g_iemAImpl_sub =
291	{
292	iemAImpl_sub_u8, iemAImpl_sub_u8_locked,
293	iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
294	iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
295	iemAImpl_sub_u64, iemAImpl_sub_u64_locked
296	};
297
298	/** Function table for the SBB instruction. */
299	static const IEMOPBINSIZES g_iemAImpl_sbb =
300	{
301	iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked,
302	iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
303	iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
304	iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
305	};
306
307	/** Function table for the OR instruction. */
308	static const IEMOPBINSIZES g_iemAImpl_or =
309	{
310	iemAImpl_or_u8, iemAImpl_or_u8_locked,
311	iemAImpl_or_u16, iemAImpl_or_u16_locked,
312	iemAImpl_or_u32, iemAImpl_or_u32_locked,
313	iemAImpl_or_u64, iemAImpl_or_u64_locked
314	};
315
316	/** Function table for the XOR instruction. */
317	static const IEMOPBINSIZES g_iemAImpl_xor =
318	{
319	iemAImpl_xor_u8, iemAImpl_xor_u8_locked,
320	iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
321	iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
322	iemAImpl_xor_u64, iemAImpl_xor_u64_locked
323	};
324
325	/** Function table for the AND instruction. */
326	static const IEMOPBINSIZES g_iemAImpl_and =
327	{
328	iemAImpl_and_u8, iemAImpl_and_u8_locked,
329	iemAImpl_and_u16, iemAImpl_and_u16_locked,
330	iemAImpl_and_u32, iemAImpl_and_u32_locked,
331	iemAImpl_and_u64, iemAImpl_and_u64_locked
332	};
333
334	/** Function table for the CMP instruction.
335	* @remarks Making operand order ASSUMPTIONS.
336	*/
337	static const IEMOPBINSIZES g_iemAImpl_cmp =
338	{
339	iemAImpl_cmp_u8, NULL,
340	iemAImpl_cmp_u16, NULL,
341	iemAImpl_cmp_u32, NULL,
342	iemAImpl_cmp_u64, NULL
343	};
344
345	/** Function table for the TEST instruction.
346	* @remarks Making operand order ASSUMPTIONS.
347	*/
348	static const IEMOPBINSIZES g_iemAImpl_test =
349	{
350	iemAImpl_test_u8, NULL,
351	iemAImpl_test_u16, NULL,
352	iemAImpl_test_u32, NULL,
353	iemAImpl_test_u64, NULL
354	};
355
356	/** Function table for the BT instruction. */
357	static const IEMOPBINSIZES g_iemAImpl_bt =
358	{
359	NULL, NULL,
360	iemAImpl_bt_u16, NULL,
361	iemAImpl_bt_u32, NULL,
362	iemAImpl_bt_u64, NULL
363	};
364
365	/** Function table for the BTC instruction. */
366	static const IEMOPBINSIZES g_iemAImpl_btc =
367	{
368	NULL, NULL,
369	iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
370	iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
371	iemAImpl_btc_u64, iemAImpl_btc_u64_locked
372	};
373
374	/** Function table for the BTR instruction. */
375	static const IEMOPBINSIZES g_iemAImpl_btr =
376	{
377	NULL, NULL,
378	iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
379	iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
380	iemAImpl_btr_u64, iemAImpl_btr_u64_locked
381	};
382
383	/** Function table for the BTS instruction. */
384	static const IEMOPBINSIZES g_iemAImpl_bts =
385	{
386	NULL, NULL,
387	iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
388	iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
389	iemAImpl_bts_u64, iemAImpl_bts_u64_locked
390	};
391
392	/** Function table for the BSF instruction. */
393	static const IEMOPBINSIZES g_iemAImpl_bsf =
394	{
395	NULL, NULL,
396	iemAImpl_bsf_u16, NULL,
397	iemAImpl_bsf_u32, NULL,
398	iemAImpl_bsf_u64, NULL
399	};
400
401	/** Function table for the BSR instruction. */
402	static const IEMOPBINSIZES g_iemAImpl_bsr =
403	{
404	NULL, NULL,
405	iemAImpl_bsr_u16, NULL,
406	iemAImpl_bsr_u32, NULL,
407	iemAImpl_bsr_u64, NULL
408	};
409
410	/** Function table for the IMUL instruction. */
411	static const IEMOPBINSIZES g_iemAImpl_imul_two =
412	{
413	NULL, NULL,
414	iemAImpl_imul_two_u16, NULL,
415	iemAImpl_imul_two_u32, NULL,
416	iemAImpl_imul_two_u64, NULL
417	};
418
419	/** Group 1 /r lookup table. */
420	static const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
421	{
422	&g_iemAImpl_add,
423	&g_iemAImpl_or,
424	&g_iemAImpl_adc,
425	&g_iemAImpl_sbb,
426	&g_iemAImpl_and,
427	&g_iemAImpl_sub,
428	&g_iemAImpl_xor,
429	&g_iemAImpl_cmp
430	};
431
432	/** Function table for the INC instruction. */
433	static const IEMOPUNARYSIZES g_iemAImpl_inc =
434	{
435	iemAImpl_inc_u8, iemAImpl_inc_u8_locked,
436	iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
437	iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
438	iemAImpl_inc_u64, iemAImpl_inc_u64_locked
439	};
440
441	/** Function table for the DEC instruction. */
442	static const IEMOPUNARYSIZES g_iemAImpl_dec =
443	{
444	iemAImpl_dec_u8, iemAImpl_dec_u8_locked,
445	iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
446	iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
447	iemAImpl_dec_u64, iemAImpl_dec_u64_locked
448	};
449
450	/** Function table for the NEG instruction. */
451	static const IEMOPUNARYSIZES g_iemAImpl_neg =
452	{
453	iemAImpl_neg_u8, iemAImpl_neg_u8_locked,
454	iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
455	iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
456	iemAImpl_neg_u64, iemAImpl_neg_u64_locked
457	};
458
459	/** Function table for the NOT instruction. */
460	static const IEMOPUNARYSIZES g_iemAImpl_not =
461	{
462	iemAImpl_not_u8, iemAImpl_not_u8_locked,
463	iemAImpl_not_u16, iemAImpl_not_u16_locked,
464	iemAImpl_not_u32, iemAImpl_not_u32_locked,
465	iemAImpl_not_u64, iemAImpl_not_u64_locked
466	};
467
468
469	/** Function table for the ROL instruction. */
470	static const IEMOPSHIFTSIZES g_iemAImpl_rol =
471	{
472	iemAImpl_rol_u8,
473	iemAImpl_rol_u16,
474	iemAImpl_rol_u32,
475	iemAImpl_rol_u64
476	};
477
478	/** Function table for the ROR instruction. */
479	static const IEMOPSHIFTSIZES g_iemAImpl_ror =
480	{
481	iemAImpl_ror_u8,
482	iemAImpl_ror_u16,
483	iemAImpl_ror_u32,
484	iemAImpl_ror_u64
485	};
486
487	/** Function table for the RCL instruction. */
488	static const IEMOPSHIFTSIZES g_iemAImpl_rcl =
489	{
490	iemAImpl_rcl_u8,
491	iemAImpl_rcl_u16,
492	iemAImpl_rcl_u32,
493	iemAImpl_rcl_u64
494	};
495
496	/** Function table for the RCR instruction. */
497	static const IEMOPSHIFTSIZES g_iemAImpl_rcr =
498	{
499	iemAImpl_rcr_u8,
500	iemAImpl_rcr_u16,
501	iemAImpl_rcr_u32,
502	iemAImpl_rcr_u64
503	};
504
505	/** Function table for the SHL instruction. */
506	static const IEMOPSHIFTSIZES g_iemAImpl_shl =
507	{
508	iemAImpl_shl_u8,
509	iemAImpl_shl_u16,
510	iemAImpl_shl_u32,
511	iemAImpl_shl_u64
512	};
513
514	/** Function table for the SHR instruction. */
515	static const IEMOPSHIFTSIZES g_iemAImpl_shr =
516	{
517	iemAImpl_shr_u8,
518	iemAImpl_shr_u16,
519	iemAImpl_shr_u32,
520	iemAImpl_shr_u64
521	};
522
523	/** Function table for the SAR instruction. */
524	static const IEMOPSHIFTSIZES g_iemAImpl_sar =
525	{
526	iemAImpl_sar_u8,
527	iemAImpl_sar_u16,
528	iemAImpl_sar_u32,
529	iemAImpl_sar_u64
530	};
531
532
533	/** Function table for the MUL instruction. */
534	static const IEMOPMULDIVSIZES g_iemAImpl_mul =
535	{
536	iemAImpl_mul_u8,
537	iemAImpl_mul_u16,
538	iemAImpl_mul_u32,
539	iemAImpl_mul_u64
540	};
541
542	/** Function table for the IMUL instruction working implicitly on rAX. */
543	static const IEMOPMULDIVSIZES g_iemAImpl_imul =
544	{
545	iemAImpl_imul_u8,
546	iemAImpl_imul_u16,
547	iemAImpl_imul_u32,
548	iemAImpl_imul_u64
549	};
550
551	/** Function table for the DIV instruction. */
552	static const IEMOPMULDIVSIZES g_iemAImpl_div =
553	{
554	iemAImpl_div_u8,
555	iemAImpl_div_u16,
556	iemAImpl_div_u32,
557	iemAImpl_div_u64
558	};
559
560	/** Function table for the MUL instruction. */
561	static const IEMOPMULDIVSIZES g_iemAImpl_idiv =
562	{
563	iemAImpl_idiv_u8,
564	iemAImpl_idiv_u16,
565	iemAImpl_idiv_u32,
566	iemAImpl_idiv_u64
567	};
568
569	/** Function table for the SHLD instruction */
570	static const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
571	{
572	iemAImpl_shld_u16,
573	iemAImpl_shld_u32,
574	iemAImpl_shld_u64,
575	};
576
577	/** Function table for the SHRD instruction */
578	static const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
579	{
580	iemAImpl_shrd_u16,
581	iemAImpl_shrd_u32,
582	iemAImpl_shrd_u64,
583	};
584
585
586	/*******************************************************************************
587	* Internal Functions *
588	*******************************************************************************/
589	static VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PIEMCPU pIemCpu);
590	/static VBOXSTRICTRC iemRaiseSelectorNotPresent(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);/
591	static VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel);
592	static VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr);
593	static VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel);
594	static VBOXSTRICTRC iemRaiseGeneralProtectionFault(PIEMCPU pIemCpu, uint16_t uErr);
595	static VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PIEMCPU pIemCpu);
596	static VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PIEMCPU pIemCpu, RTSEL uSel);
597	static VBOXSTRICTRC iemRaiseSelectorBounds(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);
598	static VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PIEMCPU pIemCpu, RTSEL Sel);
599	static VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);
600	static VBOXSTRICTRC iemRaisePageFault(PIEMCPU pIemCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
601	static VBOXSTRICTRC iemRaiseAlignmentCheckException(PIEMCPU pIemCpu);
602	static VBOXSTRICTRC iemMemMap(PIEMCPU pIemCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess);
603	static VBOXSTRICTRC iemMemCommitAndUnmap(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess);
604	static VBOXSTRICTRC iemMemFetchDataU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
605	static VBOXSTRICTRC iemMemFetchDataU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
606	static VBOXSTRICTRC iemMemFetchSysU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
607	static VBOXSTRICTRC iemMemFetchSysU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
608	static VBOXSTRICTRC iemMemFetchSelDesc(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel);
609	static VBOXSTRICTRC iemMemStackPushCommitSpecial(PIEMCPU pIemCpu, void *pvMem, uint64_t uNewRsp);
610	static VBOXSTRICTRC iemMemStackPushBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp);
611	static VBOXSTRICTRC iemMemMarkSelDescAccessed(PIEMCPU pIemCpu, uint16_t uSel);
612	static uint16_t iemSRegFetchU16(PIEMCPU pIemCpu, uint8_t iSegReg);
613
614	#ifdef IEM_VERIFICATION_MODE
615	static PIEMVERIFYEVTREC iemVerifyAllocRecord(PIEMCPU pIemCpu);
616	#endif
617	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue);
618	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue);
619
620
621	/**
622	* Initializes the decoder state.
623	*
624	* @param pIemCpu The per CPU IEM state.
625	*/
626	DECLINLINE(void) iemInitDecoder(PIEMCPU pIemCpu)
627	{
628	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
629
630	pIemCpu->uCpl = CPUMGetGuestCPL(IEMCPU_TO_VMCPU(pIemCpu), CPUMCTX2CORE(pCtx));
631	IEMMODE enmMode = CPUMIsGuestIn64BitCodeEx(pCtx)
632	? IEMMODE_64BIT
633	: pCtx->csHid.Attr.n.u1DefBig /** @todo check if this is correct... */
634	? IEMMODE_32BIT
635	: IEMMODE_16BIT;
636	pIemCpu->enmCpuMode = enmMode;
637	pIemCpu->enmDefAddrMode = enmMode; /** @todo check if this is correct... */
638	pIemCpu->enmEffAddrMode = enmMode;
639	pIemCpu->enmDefOpSize = enmMode; /** @todo check if this is correct... */
640	pIemCpu->enmEffOpSize = enmMode;
641	pIemCpu->fPrefixes = 0;
642	pIemCpu->uRexReg = 0;
643	pIemCpu->uRexB = 0;
644	pIemCpu->uRexIndex = 0;
645	pIemCpu->iEffSeg = X86_SREG_DS;
646	pIemCpu->offOpcode = 0;
647	pIemCpu->cbOpcode = 0;
648	pIemCpu->cActiveMappings = 0;
649	pIemCpu->iNextMapping = 0;
650	}
651
652
653	/**
654	* Prefetch opcodes the first time when starting executing.
655	*
656	* @returns Strict VBox status code.
657	* @param pIemCpu The IEM state.
658	*/
659	static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PIEMCPU pIemCpu)
660	{
661	#ifdef IEM_VERIFICATION_MODE
662	uint8_t const cbOldOpcodes = pIemCpu->cbOpcode;
663	#endif
664	iemInitDecoder(pIemCpu);
665
666	/*
667	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
668	*
669	* First translate CS:rIP to a physical address.
670	*/
671	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
672	uint32_t cbToTryRead;
673	RTGCPTR GCPtrPC;
674	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
675	{
676	cbToTryRead = PAGE_SIZE;
677	GCPtrPC = pCtx->rip;
678	if (!IEM_IS_CANONICAL(GCPtrPC))
679	return iemRaiseGeneralProtectionFault0(pIemCpu);
680	cbToTryRead = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
681	}
682	else
683	{
684	uint32_t GCPtrPC32 = pCtx->eip;
685	Assert(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) \|\| pIemCpu->enmCpuMode == IEMMODE_32BIT);
686	if (GCPtrPC32 > pCtx->csHid.u32Limit)
687	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
688	cbToTryRead = pCtx->csHid.u32Limit - GCPtrPC32 + 1;
689	GCPtrPC = pCtx->csHid.u64Base + GCPtrPC32;
690	}
691
692	RTGCPHYS GCPhys;
693	uint64_t fFlags;
694	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrPC, &fFlags, &GCPhys);
695	if (RT_FAILURE(rc))
696	{
697	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
698	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
699	}
700	if (!(fFlags & X86_PTE_US) && pIemCpu->uCpl == 3)
701	{
702	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
703	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
704	}
705	if ((fFlags & X86_PTE_PAE_NX) && (pCtx->msrEFER & MSR_K6_EFER_NXE))
706	{
707	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
708	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
709	}
710	GCPhys \|= GCPtrPC & PAGE_OFFSET_MASK;
711	/** @todo Check reserved bits and such stuff. PGM is better at doing
712	* that, so do it when implementing the guest virtual address
713	* TLB... */
714
715	#ifdef IEM_VERIFICATION_MODE
716	/*
717	* Optimistic optimization: Use unconsumed opcode bytes from the previous
718	* instruction.
719	*/
720	/** @todo optimize this differently by not using PGMPhysRead. */
721	RTGCPHYS const offPrevOpcodes = GCPhys - pIemCpu->GCPhysOpcodes;
722	pIemCpu->GCPhysOpcodes = GCPhys;
723	if ( offPrevOpcodes < cbOldOpcodes
724	&& PAGE_SIZE - (GCPhys & PAGE_OFFSET_MASK) > sizeof(pIemCpu->abOpcode))
725	{
726	uint8_t cbNew = cbOldOpcodes - (uint8_t)offPrevOpcodes;
727	memmove(&pIemCpu->abOpcode[0], &pIemCpu->abOpcode[offPrevOpcodes], cbNew);
728	pIemCpu->cbOpcode = cbNew;
729	return VINF_SUCCESS;
730	}
731	#endif
732
733	/*
734	* Read the bytes at this address.
735	*/
736	uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
737	if (cbToTryRead > cbLeftOnPage)
738	cbToTryRead = cbLeftOnPage;
739	if (cbToTryRead > sizeof(pIemCpu->abOpcode))
740	cbToTryRead = sizeof(pIemCpu->abOpcode);
741	/** @todo patch manager */
742	if (!pIemCpu->fByPassHandlers)
743	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhys, pIemCpu->abOpcode, cbToTryRead);
744	else
745	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pIemCpu->abOpcode, GCPhys, cbToTryRead);
746	if (rc != VINF_SUCCESS)
747	{
748	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - read error - rc=%Rrc\n", GCPtrPC, rc));
749	return rc;
750	}
751	pIemCpu->cbOpcode = cbToTryRead;
752
753	return VINF_SUCCESS;
754	}
755
756
757	/**
758	* Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
759	* exception if it fails.
760	*
761	* @returns Strict VBox status code.
762	* @param pIemCpu The IEM state.
763	* @param cbMin Where to return the opcode byte.
764	*/
765	static VBOXSTRICTRC iemOpcodeFetchMoreBytes(PIEMCPU pIemCpu, size_t cbMin)
766	{
767	/*
768	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
769	*
770	* First translate CS:rIP to a physical address.
771	*/
772	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
773	uint8_t cbLeft = pIemCpu->cbOpcode - pIemCpu->offOpcode; Assert(cbLeft < cbMin);
774	uint32_t cbToTryRead;
775	RTGCPTR GCPtrNext;
776	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
777	{
778	cbToTryRead = PAGE_SIZE;
779	GCPtrNext = pCtx->rip + pIemCpu->cbOpcode;
780	if (!IEM_IS_CANONICAL(GCPtrNext))
781	return iemRaiseGeneralProtectionFault0(pIemCpu);
782	cbToTryRead = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
783	Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
784	}
785	else
786	{
787	uint32_t GCPtrNext32 = pCtx->eip;
788	Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) \|\| pIemCpu->enmCpuMode == IEMMODE_32BIT);
789	GCPtrNext32 += pIemCpu->cbOpcode;
790	if (GCPtrNext32 > pCtx->csHid.u32Limit)
791	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
792	cbToTryRead = pCtx->csHid.u32Limit - GCPtrNext32 + 1;
793	if (cbToTryRead < cbMin - cbLeft)
794	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
795	GCPtrNext = pCtx->csHid.u64Base + GCPtrNext32;
796	}
797
798	RTGCPHYS GCPhys;
799	uint64_t fFlags;
800	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrNext, &fFlags, &GCPhys);
801	if (RT_FAILURE(rc))
802	{
803	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
804	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
805	}
806	if (!(fFlags & X86_PTE_US) && pIemCpu->uCpl == 3)
807	{
808	Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
809	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
810	}
811	if ((fFlags & X86_PTE_PAE_NX) && (pCtx->msrEFER & MSR_K6_EFER_NXE))
812	{
813	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
814	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
815	}
816	GCPhys \|= GCPtrNext & PAGE_OFFSET_MASK;
817	Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pIemCpu->cbOpcode));
818	/** @todo Check reserved bits and such stuff. PGM is better at doing
819	* that, so do it when implementing the guest virtual address
820	* TLB... */
821
822	/*
823	* Read the bytes at this address.
824	*/
825	uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
826	if (cbToTryRead > cbLeftOnPage)
827	cbToTryRead = cbLeftOnPage;
828	if (cbToTryRead > sizeof(pIemCpu->abOpcode) - pIemCpu->cbOpcode)
829	cbToTryRead = sizeof(pIemCpu->abOpcode) - pIemCpu->cbOpcode;
830	Assert(cbToTryRead >= cbMin - cbLeft);
831	if (!pIemCpu->fByPassHandlers)
832	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhys, &pIemCpu->abOpcode[pIemCpu->cbOpcode], cbToTryRead);
833	else
834	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), &pIemCpu->abOpcode[pIemCpu->cbOpcode], GCPhys, cbToTryRead);
835	if (rc != VINF_SUCCESS)
836	{
837	Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc\n", GCPtrNext, rc));
838	return rc;
839	}
840	pIemCpu->cbOpcode += cbToTryRead;
841	Log5(("%.*Rhxs\n", pIemCpu->cbOpcode, pIemCpu->abOpcode));
842
843	return VINF_SUCCESS;
844	}
845
846
847	/**
848	* Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
849	*
850	* @returns Strict VBox status code.
851	* @param pIemCpu The IEM state.
852	* @param pb Where to return the opcode byte.
853	*/
854	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU8Slow(PIEMCPU pIemCpu, uint8_t *pb)
855	{
856	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 1);
857	if (rcStrict == VINF_SUCCESS)
858	{
859	uint8_t offOpcode = pIemCpu->offOpcode;
860	*pb = pIemCpu->abOpcode[offOpcode];
861	pIemCpu->offOpcode = offOpcode + 1;
862	}
863	else
864	*pb = 0;
865	return rcStrict;
866	}
867
868
869	/**
870	* Fetches the next opcode byte.
871	*
872	* @returns Strict VBox status code.
873	* @param pIemCpu The IEM state.
874	* @param pu8 Where to return the opcode byte.
875	*/
876	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PIEMCPU pIemCpu, uint8_t *pu8)
877	{
878	uint8_t const offOpcode = pIemCpu->offOpcode;
879	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
880	return iemOpcodeGetNextU8Slow(pIemCpu, pu8);
881
882	*pu8 = pIemCpu->abOpcode[offOpcode];
883	pIemCpu->offOpcode = offOpcode + 1;
884	return VINF_SUCCESS;
885	}
886
887
888	/**
889	* Fetches the next opcode byte, returns automatically on failure.
890	*
891	* @param a_pu8 Where to return the opcode byte.
892	* @remark Implicitly references pIemCpu.
893	*/
894	#define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
895	do \
896	{ \
897	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pIemCpu, (a_pu8)); \
898	if (rcStrict2 != VINF_SUCCESS) \
899	return rcStrict2; \
900	} while (0)
901
902
903	/**
904	* Fetches the next signed byte from the opcode stream.
905	*
906	* @returns Strict VBox status code.
907	* @param pIemCpu The IEM state.
908	* @param pi8 Where to return the signed byte.
909	*/
910	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PIEMCPU pIemCpu, int8_t *pi8)
911	{
912	return iemOpcodeGetNextU8(pIemCpu, (uint8_t *)pi8);
913	}
914
915
916	/**
917	* Fetches the next signed byte from the opcode stream, returning automatically
918	* on failure.
919	*
920	* @param pi8 Where to return the signed byte.
921	* @remark Implicitly references pIemCpu.
922	*/
923	#define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
924	do \
925	{ \
926	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pIemCpu, (a_pi8)); \
927	if (rcStrict2 != VINF_SUCCESS) \
928	return rcStrict2; \
929	} while (0)
930
931
932	/**
933	* Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
934	*
935	* @returns Strict VBox status code.
936	* @param pIemCpu The IEM state.
937	* @param pu16 Where to return the opcode dword.
938	*/
939	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS8SxU16Slow(PIEMCPU pIemCpu, uint16_t *pu16)
940	{
941	uint8_t u8;
942	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pIemCpu, &u8);
943	if (rcStrict == VINF_SUCCESS)
944	*pu16 = (int8_t)u8;
945	return rcStrict;
946	}
947
948
949	/**
950	* Fetches the next signed byte from the opcode stream, extending it to
951	* unsigned 16-bit.
952	*
953	* @returns Strict VBox status code.
954	* @param pIemCpu The IEM state.
955	* @param pu16 Where to return the unsigned word.
956	*/
957	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PIEMCPU pIemCpu, uint16_t *pu16)
958	{
959	uint8_t const offOpcode = pIemCpu->offOpcode;
960	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
961	return iemOpcodeGetNextS8SxU16Slow(pIemCpu, pu16);
962
963	*pu16 = (int8_t)pIemCpu->abOpcode[offOpcode];
964	pIemCpu->offOpcode = offOpcode + 1;
965	return VINF_SUCCESS;
966	}
967
968
969	/**
970	* Fetches the next signed byte from the opcode stream and sign-extending it to
971	* a word, returning automatically on failure.
972	*
973	* @param pu16 Where to return the word.
974	* @remark Implicitly references pIemCpu.
975	*/
976	#define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
977	do \
978	{ \
979	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pIemCpu, (a_pu16)); \
980	if (rcStrict2 != VINF_SUCCESS) \
981	return rcStrict2; \
982	} while (0)
983
984
985	/**
986	* Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
987	*
988	* @returns Strict VBox status code.
989	* @param pIemCpu The IEM state.
990	* @param pu16 Where to return the opcode word.
991	*/
992	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16Slow(PIEMCPU pIemCpu, uint16_t *pu16)
993	{
994	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
995	if (rcStrict == VINF_SUCCESS)
996	{
997	uint8_t offOpcode = pIemCpu->offOpcode;
998	*pu16 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
999	pIemCpu->offOpcode = offOpcode + 2;
1000	}
1001	else
1002	*pu16 = 0;
1003	return rcStrict;
1004	}
1005
1006
1007	/**
1008	* Fetches the next opcode word.
1009	*
1010	* @returns Strict VBox status code.
1011	* @param pIemCpu The IEM state.
1012	* @param pu16 Where to return the opcode word.
1013	*/
1014	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PIEMCPU pIemCpu, uint16_t *pu16)
1015	{
1016	uint8_t const offOpcode = pIemCpu->offOpcode;
1017	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1018	return iemOpcodeGetNextU16Slow(pIemCpu, pu16);
1019
1020	*pu16 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1021	pIemCpu->offOpcode = offOpcode + 2;
1022	return VINF_SUCCESS;
1023	}
1024
1025
1026	/**
1027	* Fetches the next opcode word, returns automatically on failure.
1028	*
1029	* @param a_pu16 Where to return the opcode word.
1030	* @remark Implicitly references pIemCpu.
1031	*/
1032	#define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
1033	do \
1034	{ \
1035	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pIemCpu, (a_pu16)); \
1036	if (rcStrict2 != VINF_SUCCESS) \
1037	return rcStrict2; \
1038	} while (0)
1039
1040
1041	/**
1042	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1043	*
1044	* @returns Strict VBox status code.
1045	* @param pIemCpu The IEM state.
1046	* @param pu32 Where to return the opcode double word.
1047	*/
1048	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1049	{
1050	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1051	if (rcStrict == VINF_SUCCESS)
1052	{
1053	uint8_t offOpcode = pIemCpu->offOpcode;
1054	*pu32 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1055	pIemCpu->offOpcode = offOpcode + 2;
1056	}
1057	else
1058	*pu32 = 0;
1059	return rcStrict;
1060	}
1061
1062
1063	/**
1064	* Fetches the next opcode word, zero extending it to a double word.
1065	*
1066	* @returns Strict VBox status code.
1067	* @param pIemCpu The IEM state.
1068	* @param pu32 Where to return the opcode double word.
1069	*/
1070	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PIEMCPU pIemCpu, uint32_t *pu32)
1071	{
1072	uint8_t const offOpcode = pIemCpu->offOpcode;
1073	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1074	return iemOpcodeGetNextU16ZxU32Slow(pIemCpu, pu32);
1075
1076	*pu32 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1077	pIemCpu->offOpcode = offOpcode + 2;
1078	return VINF_SUCCESS;
1079	}
1080
1081
1082	/**
1083	* Fetches the next opcode word and zero extends it to a double word, returns
1084	* automatically on failure.
1085	*
1086	* @param a_pu32 Where to return the opcode double word.
1087	* @remark Implicitly references pIemCpu.
1088	*/
1089	#define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1090	do \
1091	{ \
1092	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pIemCpu, (a_pu32)); \
1093	if (rcStrict2 != VINF_SUCCESS) \
1094	return rcStrict2; \
1095	} while (0)
1096
1097
1098	/**
1099	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1100	*
1101	* @returns Strict VBox status code.
1102	* @param pIemCpu The IEM state.
1103	* @param pu64 Where to return the opcode quad word.
1104	*/
1105	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1106	{
1107	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1108	if (rcStrict == VINF_SUCCESS)
1109	{
1110	uint8_t offOpcode = pIemCpu->offOpcode;
1111	*pu64 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1112	pIemCpu->offOpcode = offOpcode + 2;
1113	}
1114	else
1115	*pu64 = 0;
1116	return rcStrict;
1117	}
1118
1119
1120	/**
1121	* Fetches the next opcode word, zero extending it to a quad word.
1122	*
1123	* @returns Strict VBox status code.
1124	* @param pIemCpu The IEM state.
1125	* @param pu64 Where to return the opcode quad word.
1126	*/
1127	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1128	{
1129	uint8_t const offOpcode = pIemCpu->offOpcode;
1130	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1131	return iemOpcodeGetNextU16ZxU64Slow(pIemCpu, pu64);
1132
1133	*pu64 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1134	pIemCpu->offOpcode = offOpcode + 2;
1135	return VINF_SUCCESS;
1136	}
1137
1138
1139	/**
1140	* Fetches the next opcode word and zero extends it to a quad word, returns
1141	* automatically on failure.
1142	*
1143	* @param a_pu64 Where to return the opcode quad word.
1144	* @remark Implicitly references pIemCpu.
1145	*/
1146	#define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1147	do \
1148	{ \
1149	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pIemCpu, (a_pu64)); \
1150	if (rcStrict2 != VINF_SUCCESS) \
1151	return rcStrict2; \
1152	} while (0)
1153
1154
1155	/**
1156	* Fetches the next signed word from the opcode stream.
1157	*
1158	* @returns Strict VBox status code.
1159	* @param pIemCpu The IEM state.
1160	* @param pi16 Where to return the signed word.
1161	*/
1162	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PIEMCPU pIemCpu, int16_t *pi16)
1163	{
1164	return iemOpcodeGetNextU16(pIemCpu, (uint16_t *)pi16);
1165	}
1166
1167
1168	/**
1169	* Fetches the next signed word from the opcode stream, returning automatically
1170	* on failure.
1171	*
1172	* @param pi16 Where to return the signed word.
1173	* @remark Implicitly references pIemCpu.
1174	*/
1175	#define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1176	do \
1177	{ \
1178	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pIemCpu, (a_pi16)); \
1179	if (rcStrict2 != VINF_SUCCESS) \
1180	return rcStrict2; \
1181	} while (0)
1182
1183
1184	/**
1185	* Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1186	*
1187	* @returns Strict VBox status code.
1188	* @param pIemCpu The IEM state.
1189	* @param pu32 Where to return the opcode dword.
1190	*/
1191	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1192	{
1193	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1194	if (rcStrict == VINF_SUCCESS)
1195	{
1196	uint8_t offOpcode = pIemCpu->offOpcode;
1197	*pu32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1198	pIemCpu->abOpcode[offOpcode + 1],
1199	pIemCpu->abOpcode[offOpcode + 2],
1200	pIemCpu->abOpcode[offOpcode + 3]);
1201	pIemCpu->offOpcode = offOpcode + 4;
1202	}
1203	else
1204	*pu32 = 0;
1205	return rcStrict;
1206	}
1207
1208
1209	/**
1210	* Fetches the next opcode dword.
1211	*
1212	* @returns Strict VBox status code.
1213	* @param pIemCpu The IEM state.
1214	* @param pu32 Where to return the opcode double word.
1215	*/
1216	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PIEMCPU pIemCpu, uint32_t *pu32)
1217	{
1218	uint8_t const offOpcode = pIemCpu->offOpcode;
1219	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1220	return iemOpcodeGetNextU32Slow(pIemCpu, pu32);
1221
1222	*pu32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1223	pIemCpu->abOpcode[offOpcode + 1],
1224	pIemCpu->abOpcode[offOpcode + 2],
1225	pIemCpu->abOpcode[offOpcode + 3]);
1226	pIemCpu->offOpcode = offOpcode + 4;
1227	return VINF_SUCCESS;
1228	}
1229
1230
1231	/**
1232	* Fetches the next opcode dword, returns automatically on failure.
1233	*
1234	* @param a_pu32 Where to return the opcode dword.
1235	* @remark Implicitly references pIemCpu.
1236	*/
1237	#define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1238	do \
1239	{ \
1240	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pIemCpu, (a_pu32)); \
1241	if (rcStrict2 != VINF_SUCCESS) \
1242	return rcStrict2; \
1243	} while (0)
1244
1245
1246	/**
1247	* Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1248	*
1249	* @returns Strict VBox status code.
1250	* @param pIemCpu The IEM state.
1251	* @param pu32 Where to return the opcode dword.
1252	*/
1253	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1254	{
1255	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1256	if (rcStrict == VINF_SUCCESS)
1257	{
1258	uint8_t offOpcode = pIemCpu->offOpcode;
1259	*pu64 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1260	pIemCpu->abOpcode[offOpcode + 1],
1261	pIemCpu->abOpcode[offOpcode + 2],
1262	pIemCpu->abOpcode[offOpcode + 3]);
1263	pIemCpu->offOpcode = offOpcode + 4;
1264	}
1265	else
1266	*pu64 = 0;
1267	return rcStrict;
1268	}
1269
1270
1271	/**
1272	* Fetches the next opcode dword, zero extending it to a quad word.
1273	*
1274	* @returns Strict VBox status code.
1275	* @param pIemCpu The IEM state.
1276	* @param pu64 Where to return the opcode quad word.
1277	*/
1278	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1279	{
1280	uint8_t const offOpcode = pIemCpu->offOpcode;
1281	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1282	return iemOpcodeGetNextU32ZxU64Slow(pIemCpu, pu64);
1283
1284	*pu64 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1285	pIemCpu->abOpcode[offOpcode + 1],
1286	pIemCpu->abOpcode[offOpcode + 2],
1287	pIemCpu->abOpcode[offOpcode + 3]);
1288	pIemCpu->offOpcode = offOpcode + 4;
1289	return VINF_SUCCESS;
1290	}
1291
1292
1293	/**
1294	* Fetches the next opcode dword and zero extends it to a quad word, returns
1295	* automatically on failure.
1296	*
1297	* @param a_pu64 Where to return the opcode quad word.
1298	* @remark Implicitly references pIemCpu.
1299	*/
1300	#define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1301	do \
1302	{ \
1303	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pIemCpu, (a_pu64)); \
1304	if (rcStrict2 != VINF_SUCCESS) \
1305	return rcStrict2; \
1306	} while (0)
1307
1308
1309	/**
1310	* Fetches the next signed double word from the opcode stream.
1311	*
1312	* @returns Strict VBox status code.
1313	* @param pIemCpu The IEM state.
1314	* @param pi32 Where to return the signed double word.
1315	*/
1316	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PIEMCPU pIemCpu, int32_t *pi32)
1317	{
1318	return iemOpcodeGetNextU32(pIemCpu, (uint32_t *)pi32);
1319	}
1320
1321	/**
1322	* Fetches the next signed double word from the opcode stream, returning
1323	* automatically on failure.
1324	*
1325	* @param pi32 Where to return the signed double word.
1326	* @remark Implicitly references pIemCpu.
1327	*/
1328	#define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1329	do \
1330	{ \
1331	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pIemCpu, (a_pi32)); \
1332	if (rcStrict2 != VINF_SUCCESS) \
1333	return rcStrict2; \
1334	} while (0)
1335
1336
1337	/**
1338	* Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1339	*
1340	* @returns Strict VBox status code.
1341	* @param pIemCpu The IEM state.
1342	* @param pu64 Where to return the opcode qword.
1343	*/
1344	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS32SxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1345	{
1346	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1347	if (rcStrict == VINF_SUCCESS)
1348	{
1349	uint8_t offOpcode = pIemCpu->offOpcode;
1350	*pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1351	pIemCpu->abOpcode[offOpcode + 1],
1352	pIemCpu->abOpcode[offOpcode + 2],
1353	pIemCpu->abOpcode[offOpcode + 3]);
1354	pIemCpu->offOpcode = offOpcode + 4;
1355	}
1356	else
1357	*pu64 = 0;
1358	return rcStrict;
1359	}
1360
1361
1362	/**
1363	* Fetches the next opcode dword, sign extending it into a quad word.
1364	*
1365	* @returns Strict VBox status code.
1366	* @param pIemCpu The IEM state.
1367	* @param pu64 Where to return the opcode quad word.
1368	*/
1369	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1370	{
1371	uint8_t const offOpcode = pIemCpu->offOpcode;
1372	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1373	return iemOpcodeGetNextS32SxU64Slow(pIemCpu, pu64);
1374
1375	int32_t i32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1376	pIemCpu->abOpcode[offOpcode + 1],
1377	pIemCpu->abOpcode[offOpcode + 2],
1378	pIemCpu->abOpcode[offOpcode + 3]);
1379	*pu64 = i32;
1380	pIemCpu->offOpcode = offOpcode + 4;
1381	return VINF_SUCCESS;
1382	}
1383
1384
1385	/**
1386	* Fetches the next opcode double word and sign extends it to a quad word,
1387	* returns automatically on failure.
1388	*
1389	* @param a_pu64 Where to return the opcode quad word.
1390	* @remark Implicitly references pIemCpu.
1391	*/
1392	#define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1393	do \
1394	{ \
1395	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pIemCpu, (a_pu64)); \
1396	if (rcStrict2 != VINF_SUCCESS) \
1397	return rcStrict2; \
1398	} while (0)
1399
1400
1401	/**
1402	* Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1403	*
1404	* @returns Strict VBox status code.
1405	* @param pIemCpu The IEM state.
1406	* @param pu64 Where to return the opcode qword.
1407	*/
1408	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1409	{
1410	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 8);
1411	if (rcStrict == VINF_SUCCESS)
1412	{
1413	uint8_t offOpcode = pIemCpu->offOpcode;
1414	*pu64 = RT_MAKE_U64_FROM_U8(pIemCpu->abOpcode[offOpcode],
1415	pIemCpu->abOpcode[offOpcode + 1],
1416	pIemCpu->abOpcode[offOpcode + 2],
1417	pIemCpu->abOpcode[offOpcode + 3],
1418	pIemCpu->abOpcode[offOpcode + 4],
1419	pIemCpu->abOpcode[offOpcode + 5],
1420	pIemCpu->abOpcode[offOpcode + 6],
1421	pIemCpu->abOpcode[offOpcode + 7]);
1422	pIemCpu->offOpcode = offOpcode + 8;
1423	}
1424	else
1425	*pu64 = 0;
1426	return rcStrict;
1427	}
1428
1429
1430	/**
1431	* Fetches the next opcode qword.
1432	*
1433	* @returns Strict VBox status code.
1434	* @param pIemCpu The IEM state.
1435	* @param pu64 Where to return the opcode qword.
1436	*/
1437	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PIEMCPU pIemCpu, uint64_t *pu64)
1438	{
1439	uint8_t const offOpcode = pIemCpu->offOpcode;
1440	if (RT_UNLIKELY(offOpcode + 8 > pIemCpu->cbOpcode))
1441	return iemOpcodeGetNextU64Slow(pIemCpu, pu64);
1442
1443	*pu64 = RT_MAKE_U64_FROM_U8(pIemCpu->abOpcode[offOpcode],
1444	pIemCpu->abOpcode[offOpcode + 1],
1445	pIemCpu->abOpcode[offOpcode + 2],
1446	pIemCpu->abOpcode[offOpcode + 3],
1447	pIemCpu->abOpcode[offOpcode + 4],
1448	pIemCpu->abOpcode[offOpcode + 5],
1449	pIemCpu->abOpcode[offOpcode + 6],
1450	pIemCpu->abOpcode[offOpcode + 7]);
1451	pIemCpu->offOpcode = offOpcode + 8;
1452	return VINF_SUCCESS;
1453	}
1454
1455
1456	/**
1457	* Fetches the next opcode quad word, returns automatically on failure.
1458	*
1459	* @param a_pu64 Where to return the opcode quad word.
1460	* @remark Implicitly references pIemCpu.
1461	*/
1462	#define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1463	do \
1464	{ \
1465	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pIemCpu, (a_pu64)); \
1466	if (rcStrict2 != VINF_SUCCESS) \
1467	return rcStrict2; \
1468	} while (0)
1469
1470
1471	/** @name Misc Worker Functions.
1472	* @{
1473	*/
1474
1475
1476	/**
1477	* Validates a new SS segment.
1478	*
1479	* @returns VBox strict status code.
1480	* @param pIemCpu The IEM per CPU instance data.
1481	* @param pCtx The CPU context.
1482	* @param NewSS The new SS selctor.
1483	* @param uCpl The CPL to load the stack for.
1484	* @param pDesc Where to return the descriptor.
1485	*/
1486	static VBOXSTRICTRC iemMiscValidateNewSS(PIEMCPU pIemCpu, PCCPUMCTX pCtx, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc)
1487	{
1488	NOREF(pCtx);
1489
1490	/* Null selectors are not allowed (we're not called for dispatching
1491	interrupts with SS=0 in long mode). */
1492	if (!(NewSS & (X86_SEL_MASK \| X86_SEL_LDT)))
1493	{
1494	Log(("iemMiscValidateNewSSandRsp: #x - null selector -> #GP(0)\n", NewSS));
1495	return iemRaiseGeneralProtectionFault0(pIemCpu);
1496	}
1497
1498	/*
1499	* Read the descriptor.
1500	*/
1501	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pIemCpu, pDesc, NewSS);
1502	if (rcStrict != VINF_SUCCESS)
1503	return rcStrict;
1504
1505	/*
1506	* Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1507	*/
1508	if (!pDesc->Legacy.Gen.u1DescType)
1509	{
1510	Log(("iemMiscValidateNewSSandRsp: %#x - system selector -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1511	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1512	}
1513
1514	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1515	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1516	{
1517	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1518	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1519	}
1520	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1521	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1522	{
1523	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1524	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1525	}
1526	/** @todo testcase: check if the TSS.ssX RPL is checked. */
1527	if ((NewSS & X86_SEL_RPL) != uCpl)
1528	{
1529	Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #GP\n", NewSS, uCpl));
1530	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1531	}
1532	if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1533	{
1534	Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #GP\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1535	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1536	}
1537
1538	/* Is it there? */
1539	/** @todo testcase: Is this checked before the canonical / limit check below? */
1540	if (!pDesc->Legacy.Gen.u1Present)
1541	{
1542	Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1543	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewSS);
1544	}
1545
1546	return VINF_SUCCESS;
1547	}
1548
1549
1550	/** @} */
1551
1552	/** @name Raising Exceptions.
1553	*
1554	* @{
1555	*/
1556
1557	/** @name IEM_XCPT_FLAGS_XXX - flags for iemRaiseXcptOrInt.
1558	* @{ */
1559	/** CPU exception. */
1560	#define IEM_XCPT_FLAGS_T_CPU_XCPT RT_BIT_32(0)
1561	/** External interrupt (from PIC, APIC, whatever). */
1562	#define IEM_XCPT_FLAGS_T_EXT_INT RT_BIT_32(1)
1563	/** Software interrupt (int, into or bound). */
1564	#define IEM_XCPT_FLAGS_T_SOFT_INT RT_BIT_32(2)
1565	/** Takes an error code. */
1566	#define IEM_XCPT_FLAGS_ERR RT_BIT_32(3)
1567	/** Takes a CR2. */
1568	#define IEM_XCPT_FLAGS_CR2 RT_BIT_32(4)
1569	/** Generated by the breakpoint instruction. */
1570	#define IEM_XCPT_FLAGS_BP_INSTR RT_BIT_32(5)
1571	/** @} */
1572
1573	/**
1574	* Loads the specified stack far pointer from the TSS.
1575	*
1576	* @returns VBox strict status code.
1577	* @param pIemCpu The IEM per CPU instance data.
1578	* @param pCtx The CPU context.
1579	* @param uCpl The CPL to load the stack for.
1580	* @param pSelSS Where to return the new stack segment.
1581	* @param puEsp Where to return the new stack pointer.
1582	*/
1583	static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PIEMCPU pIemCpu, PCCPUMCTX pCtx, uint8_t uCpl,
1584	PRTSEL pSelSS, uint32_t *puEsp)
1585	{
1586	VBOXSTRICTRC rcStrict;
1587	Assert(uCpl < 4);
1588	puEsp = 0; / make gcc happy */
1589	pSelSS = 0; / make gcc happy */
1590
1591	switch (pCtx->trHid.Attr.n.u4Type)
1592	{
1593	/*
1594	* 16-bit TSS (X86TSS16).
1595	*/
1596	case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed();
1597	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1598	{
1599	uint32_t off = uCpl * 4 + 2;
1600	if (off + 4 > pCtx->trHid.u32Limit)
1601	{
1602	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pCtx->trHid.u32Limit));
1603	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
1604	}
1605
1606	uint32_t u32Tmp = 0; /* gcc maybe... */
1607	rcStrict = iemMemFetchSysU32(pIemCpu, &u32Tmp, UINT8_MAX, pCtx->trHid.u64Base + off);
1608	if (rcStrict == VINF_SUCCESS)
1609	{
1610	*puEsp = RT_LOWORD(u32Tmp);
1611	*pSelSS = RT_HIWORD(u32Tmp);
1612	return VINF_SUCCESS;
1613	}
1614	break;
1615	}
1616
1617	/*
1618	* 32-bit TSS (X86TSS32).
1619	*/
1620	case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed();
1621	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1622	{
1623	uint32_t off = uCpl * 8 + 4;
1624	if (off + 7 > pCtx->trHid.u32Limit)
1625	{
1626	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pCtx->trHid.u32Limit));
1627	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
1628	}
1629
1630	uint64_t u64Tmp;
1631	rcStrict = iemMemFetchSysU64(pIemCpu, &u64Tmp, UINT8_MAX, pCtx->trHid.u64Base + off);
1632	if (rcStrict == VINF_SUCCESS)
1633	{
1634	*puEsp = u64Tmp & UINT32_MAX;
1635	*pSelSS = (RTSEL)(u64Tmp >> 32);
1636	return VINF_SUCCESS;
1637	}
1638	break;
1639	}
1640
1641	default:
1642	AssertFailedReturn(VERR_INTERNAL_ERROR_2);
1643	}
1644	return rcStrict;
1645	}
1646
1647
1648	/**
1649	* Adjust the CPU state according to the exception being raised.
1650	*
1651	* @param pCtx The CPU context.
1652	* @param u8Vector The exception that has been raised.
1653	*/
1654	DECLINLINE(void) iemRaiseXcptAdjustState(PCPUMCTX pCtx, uint8_t u8Vector)
1655	{
1656	switch (u8Vector)
1657	{
1658	case X86_XCPT_DB:
1659	pCtx->dr[7] &= ~X86_DR7_GD;
1660	break;
1661	/** @todo Read the AMD and Intel exception reference... */
1662	}
1663	}
1664
1665
1666	/**
1667	* Implements exceptions and interrupts for real mode.
1668	*
1669	* @returns VBox strict status code.
1670	* @param pIemCpu The IEM per CPU instance data.
1671	* @param pCtx The CPU context.
1672	* @param cbInstr The number of bytes to offset rIP by in the return
1673	* address.
1674	* @param u8Vector The interrupt / exception vector number.
1675	* @param fFlags The flags.
1676	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1677	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1678	*/
1679	static VBOXSTRICTRC
1680	iemRaiseXcptOrIntInRealMode(PIEMCPU pIemCpu,
1681	PCPUMCTX pCtx,
1682	uint8_t cbInstr,
1683	uint8_t u8Vector,
1684	uint32_t fFlags,
1685	uint16_t uErr,
1686	uint64_t uCr2)
1687	{
1688	AssertReturn(pIemCpu->enmCpuMode == IEMMODE_16BIT, VERR_INTERNAL_ERROR_3);
1689	NOREF(uErr); NOREF(uCr2);
1690
1691	/*
1692	* Read the IDT entry.
1693	*/
1694	if (pCtx->idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
1695	{
1696	Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
1697	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1698	}
1699	RTFAR16 Idte;
1700	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pIemCpu, (uint32_t *)&Idte, UINT8_MAX,
1701	pCtx->idtr.pIdt + UINT32_C(4) * u8Vector);
1702	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1703	return rcStrict;
1704
1705	/*
1706	* Push the stack frame.
1707	*/
1708	uint16_t *pu16Frame;
1709	uint64_t uNewRsp;
1710	rcStrict = iemMemStackPushBeginSpecial(pIemCpu, 6, (void **)&pu16Frame, &uNewRsp);
1711	if (rcStrict != VINF_SUCCESS)
1712	return rcStrict;
1713
1714	pu16Frame[2] = (uint16_t)pCtx->eflags.u;
1715	pu16Frame[1] = (uint16_t)pCtx->cs;
1716	pu16Frame[0] = pCtx->ip + cbInstr;
1717	rcStrict = iemMemStackPushCommitSpecial(pIemCpu, pu16Frame, uNewRsp);
1718	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1719	return rcStrict;
1720
1721	/*
1722	* Load the vector address into cs:ip and make exception specific state
1723	* adjustments.
1724	*/
1725	pCtx->cs = Idte.sel;
1726	pCtx->csHid.u64Base = (uint32_t)Idte.sel << 4;
1727	/** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
1728	pCtx->rip = Idte.off;
1729	pCtx->eflags.Bits.u1IF = 0;
1730
1731	/** @todo do we actually do this in real mode? */
1732	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1733	iemRaiseXcptAdjustState(pCtx, u8Vector);
1734
1735	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
1736	}
1737
1738
1739	/**
1740	* Implements exceptions and interrupts for protected mode.
1741	*
1742	* @returns VBox strict status code.
1743	* @param pIemCpu The IEM per CPU instance data.
1744	* @param pCtx The CPU context.
1745	* @param cbInstr The number of bytes to offset rIP by in the return
1746	* address.
1747	* @param u8Vector The interrupt / exception vector number.
1748	* @param fFlags The flags.
1749	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1750	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1751	*/
1752	static VBOXSTRICTRC
1753	iemRaiseXcptOrIntInProtMode(PIEMCPU pIemCpu,
1754	PCPUMCTX pCtx,
1755	uint8_t cbInstr,
1756	uint8_t u8Vector,
1757	uint32_t fFlags,
1758	uint16_t uErr,
1759	uint64_t uCr2)
1760	{
1761	NOREF(cbInstr);
1762
1763	/*
1764	* Read the IDT entry.
1765	*/
1766	if (pCtx->idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
1767	{
1768	Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
1769	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1770	}
1771	X86DESC Idte;
1772	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &Idte.u, UINT8_MAX,
1773	pCtx->idtr.pIdt + UINT32_C(8) * u8Vector);
1774	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1775	return rcStrict;
1776	LogFlow(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
1777	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
1778	Idte.Gate.u4ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
1779
1780	/*
1781	* Check the descriptor type, DPL and such.
1782	* ASSUMES this is done in the same order as described for call-gate calls.
1783	*/
1784	if (Idte.Gate.u1DescType)
1785	{
1786	Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
1787	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1788	}
1789	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
1790	switch (Idte.Gate.u4Type)
1791	{
1792	case X86_SEL_TYPE_SYS_UNDEFINED:
1793	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
1794	case X86_SEL_TYPE_SYS_LDT:
1795	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1796	case X86_SEL_TYPE_SYS_286_CALL_GATE:
1797	case X86_SEL_TYPE_SYS_UNDEFINED2:
1798	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
1799	case X86_SEL_TYPE_SYS_UNDEFINED3:
1800	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1801	case X86_SEL_TYPE_SYS_386_CALL_GATE:
1802	case X86_SEL_TYPE_SYS_UNDEFINED4:
1803	{
1804	/** @todo check what actually happens when the type is wrong...
1805	* esp. call gates. */
1806	Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
1807	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1808	}
1809
1810	case X86_SEL_TYPE_SYS_286_INT_GATE:
1811	case X86_SEL_TYPE_SYS_386_INT_GATE:
1812	fEflToClear \|= X86_EFL_IF;
1813	break;
1814
1815	case X86_SEL_TYPE_SYS_TASK_GATE:
1816	/** @todo task gates. */
1817	AssertFailedReturn(VERR_NOT_SUPPORTED);
1818
1819	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
1820	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
1821	break;
1822
1823	IEM_NOT_REACHED_DEFAULT_CASE_RET();
1824	}
1825
1826	/* Check DPL against CPL if applicable. */
1827	if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
1828	{
1829	if (pIemCpu->uCpl > Idte.Gate.u2Dpl)
1830	{
1831	Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pIemCpu->uCpl, Idte.Gate.u2Dpl));
1832	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1833	}
1834	}
1835
1836	/* Is it there? */
1837	if (!Idte.Gate.u1Present)
1838	{
1839	Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
1840	return iemRaiseSelectorNotPresentWithErr(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1841	}
1842
1843	/* A null CS is bad. */
1844	RTSEL NewCS = Idte.Gate.u16Sel;
1845	if (!(NewCS & (X86_SEL_MASK \| X86_SEL_LDT)))
1846	{
1847	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
1848	return iemRaiseGeneralProtectionFault0(pIemCpu);
1849	}
1850
1851	/* Fetch the descriptor for the new CS. */
1852	IEMSELDESC DescCS;
1853	rcStrict = iemMemFetchSelDesc(pIemCpu, &DescCS, NewCS);
1854	if (rcStrict != VINF_SUCCESS)
1855	{
1856	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
1857	return rcStrict;
1858	}
1859
1860	/* Must be a code segment. */
1861	if (!DescCS.Legacy.Gen.u1DescType)
1862	{
1863	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
1864	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1865	}
1866	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
1867	{
1868	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
1869	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1870	}
1871
1872	/* Don't allow lowering the privilege level. */
1873	/** @todo Does the lowering of privileges apply to software interrupts
1874	* only? This has bearings on the more-privileged or
1875	* same-privilege stack behavior further down. A testcase would
1876	* be nice. */
1877	if (DescCS.Legacy.Gen.u2Dpl > pIemCpu->uCpl)
1878	{
1879	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
1880	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
1881	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1882	}
1883	/** @todo is the RPL of the interrupt/trap gate descriptor checked? */
1884
1885	/* Check the new EIP against the new CS limit. */
1886	uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
1887	\|\| Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
1888	? Idte.Gate.u16OffsetLow
1889	: Idte.Gate.u16OffsetLow \| ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
1890	uint32_t cbLimitCS = X86DESC_LIMIT(DescCS.Legacy);
1891	if (DescCS.Legacy.Gen.u1Granularity)
1892	cbLimitCS = (cbLimitCS << PAGE_SHIFT) \| PAGE_OFFSET_MASK;
1893	if (uNewEip > cbLimitCS)
1894	{
1895	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
1896	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
1897	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1898	}
1899
1900	/* Make sure the selector is present. */
1901	if (!DescCS.Legacy.Gen.u1Present)
1902	{
1903	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
1904	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewCS);
1905	}
1906
1907	/*
1908	* If the privilege level changes, we need to get a new stack from the TSS.
1909	* This in turns means validating the new SS and ESP...
1910	*/
1911	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
1912	? pIemCpu->uCpl : DescCS.Legacy.Gen.u2Dpl;
1913	if (uNewCpl != pIemCpu->uCpl)
1914	{
1915	RTSEL NewSS;
1916	uint32_t uNewEsp;
1917	rcStrict = iemRaiseLoadStackFromTss32Or16(pIemCpu, pCtx, uNewCpl, &NewSS, &uNewEsp);
1918	if (rcStrict != VINF_SUCCESS)
1919	return rcStrict;
1920
1921	IEMSELDESC DescSS;
1922	rcStrict = iemMiscValidateNewSS(pIemCpu, pCtx, NewSS, uNewCpl, &DescSS);
1923	if (rcStrict != VINF_SUCCESS)
1924	return rcStrict;
1925
1926	/* Check that there is sufficient space for the stack frame. */
1927	uint32_t cbLimitSS = X86DESC_LIMIT(DescSS.Legacy);
1928	if (DescSS.Legacy.Gen.u1Granularity)
1929	cbLimitSS = (cbLimitSS << PAGE_SHIFT) \| PAGE_OFFSET_MASK;
1930	AssertReturn(!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
1931
1932	uint8_t const cbStackFrame = fFlags & IEM_XCPT_FLAGS_ERR ? 24 : 20;
1933	if ( uNewEsp - 1 > cbLimitSS
1934	\|\| uNewEsp < cbStackFrame)
1935	{
1936	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
1937	u8Vector, NewSS, uNewEsp, cbStackFrame));
1938	return iemRaiseSelectorBoundsBySelector(pIemCpu, NewSS);
1939	}
1940
1941	/*
1942	* Start making changes.
1943	*/
1944
1945	/* Create the stack frame. */
1946	RTPTRUNION uStackFrame;
1947	rcStrict = iemMemMap(pIemCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
1948	uNewEsp - cbStackFrame + X86DESC_BASE(DescSS.Legacy), IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
1949	if (rcStrict != VINF_SUCCESS)
1950	return rcStrict;
1951	void * const pvStackFrame = uStackFrame.pv;
1952
1953	if (fFlags & IEM_XCPT_FLAGS_ERR)
1954	*uStackFrame.pu32++ = uErr;
1955	uStackFrame.pu32[0] = (fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
1956	? pCtx->eip + cbInstr : pCtx->eip;
1957	uStackFrame.pu32[1] = (pCtx->cs & ~X86_SEL_RPL) \| pIemCpu->uCpl;
1958	uStackFrame.pu32[2] = pCtx->eflags.u;
1959	uStackFrame.pu32[3] = pCtx->esp;
1960	uStackFrame.pu32[4] = pCtx->ss;
1961	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
1962	if (rcStrict != VINF_SUCCESS)
1963	return rcStrict;
1964
1965	/* Mark the selectors 'accessed' (hope this is the correct time). */
1966	/** @todo testcase: excatly _when_ are the accessed bits set - before or
1967	* after pushing the stack frame? (Write protect the gdt + stack to
1968	* find out.) */
1969	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1970	{
1971	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
1972	if (rcStrict != VINF_SUCCESS)
1973	return rcStrict;
1974	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1975	}
1976
1977	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1978	{
1979	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewSS);
1980	if (rcStrict != VINF_SUCCESS)
1981	return rcStrict;
1982	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1983	}
1984
1985	/*
1986	* Start commint the register changes (joins with the DPL=CPL branch).
1987	*/
1988	pCtx->ss = NewSS;
1989	pCtx->ssHid.u32Limit = cbLimitSS;
1990	pCtx->ssHid.u64Base = X86DESC_BASE(DescSS.Legacy);
1991	pCtx->ssHid.Attr.u = X86DESC_GET_HID_ATTR(DescSS.Legacy);
1992	pCtx->rsp = uNewEsp - cbStackFrame; /** @todo Is the high word cleared for 16-bit stacks and/or interrupt handlers? */
1993	pIemCpu->uCpl = uNewCpl;
1994	}
1995	/*
1996	* Same privilege, no stack change and smaller stack frame.
1997	*/
1998	else
1999	{
2000	uint64_t uNewRsp;
2001	RTPTRUNION uStackFrame;
2002	uint8_t const cbStackFrame = fFlags & IEM_XCPT_FLAGS_ERR ? 16 : 12;
2003	rcStrict = iemMemStackPushBeginSpecial(pIemCpu, cbStackFrame, &uStackFrame.pv, &uNewRsp);
2004	if (rcStrict != VINF_SUCCESS)
2005	return rcStrict;
2006	void * const pvStackFrame = uStackFrame.pv;
2007
2008	if (fFlags & IEM_XCPT_FLAGS_ERR)
2009	*uStackFrame.pu32++ = uErr;
2010	uStackFrame.pu32[0] = (fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
2011	? pCtx->eip + cbInstr : pCtx->eip;
2012	uStackFrame.pu32[1] = (pCtx->cs & ~X86_SEL_RPL) \| pIemCpu->uCpl;
2013	uStackFrame.pu32[2] = pCtx->eflags.u;
2014	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
2015	if (rcStrict != VINF_SUCCESS)
2016	return rcStrict;
2017
2018	/* Mark the CS selector as 'accessed'. */
2019	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2020	{
2021	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
2022	if (rcStrict != VINF_SUCCESS)
2023	return rcStrict;
2024	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2025	}
2026
2027	/*
2028	* Start committing the register changes (joins with the other branch).
2029	*/
2030	pCtx->rsp = uNewRsp;
2031	}
2032
2033	/* ... register committing continues. */
2034	pCtx->cs = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
2035	pCtx->csHid.u32Limit = cbLimitCS;
2036	pCtx->csHid.u64Base = X86DESC_BASE(DescCS.Legacy);
2037	pCtx->csHid.Attr.u = X86DESC_GET_HID_ATTR(DescCS.Legacy);
2038
2039	pCtx->rip = uNewEip;
2040	pCtx->rflags.u &= ~fEflToClear;
2041
2042	if (fFlags & IEM_XCPT_FLAGS_CR2)
2043	pCtx->cr2 = uCr2;
2044
2045	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
2046	iemRaiseXcptAdjustState(pCtx, u8Vector);
2047
2048	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2049	}
2050
2051
2052	/**
2053	* Implements exceptions and interrupts for V8086 mode.
2054	*
2055	* @returns VBox strict status code.
2056	* @param pIemCpu The IEM per CPU instance data.
2057	* @param pCtx The CPU context.
2058	* @param cbInstr The number of bytes to offset rIP by in the return
2059	* address.
2060	* @param u8Vector The interrupt / exception vector number.
2061	* @param fFlags The flags.
2062	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2063	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2064	*/
2065	static VBOXSTRICTRC
2066	iemRaiseXcptOrIntInV8086Mode(PIEMCPU pIemCpu,
2067	PCPUMCTX pCtx,
2068	uint8_t cbInstr,
2069	uint8_t u8Vector,
2070	uint32_t fFlags,
2071	uint16_t uErr,
2072	uint64_t uCr2)
2073	{
2074	NOREF(pIemCpu); NOREF(pCtx); NOREF(cbInstr); NOREF(u8Vector); NOREF(fFlags); NOREF(uErr); NOREF(uCr2);
2075	AssertMsgFailed(("V8086 exception / interrupt dispatching\n"));
2076	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
2077	}
2078
2079
2080	/**
2081	* Implements exceptions and interrupts for long mode.
2082	*
2083	* @returns VBox strict status code.
2084	* @param pIemCpu The IEM per CPU instance data.
2085	* @param pCtx The CPU context.
2086	* @param cbInstr The number of bytes to offset rIP by in the return
2087	* address.
2088	* @param u8Vector The interrupt / exception vector number.
2089	* @param fFlags The flags.
2090	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2091	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2092	*/
2093	static VBOXSTRICTRC
2094	iemRaiseXcptOrIntInLongMode(PIEMCPU pIemCpu,
2095	PCPUMCTX pCtx,
2096	uint8_t cbInstr,
2097	uint8_t u8Vector,
2098	uint32_t fFlags,
2099	uint16_t uErr,
2100	uint64_t uCr2)
2101	{
2102	NOREF(pIemCpu); NOREF(pCtx); NOREF(cbInstr); NOREF(u8Vector); NOREF(fFlags); NOREF(uErr); NOREF(uCr2);
2103	AssertMsgFailed(("long mode exception / interrupt dispatching\n"));
2104	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
2105	}
2106
2107
2108	/**
2109	* Implements exceptions and interrupts.
2110	*
2111	* All exceptions and interrupts goes thru this function!
2112	*
2113	* @returns VBox strict status code.
2114	* @param pIemCpu The IEM per CPU instance data.
2115	* @param cbInstr The number of bytes to offset rIP by in the return
2116	* address.
2117	* @param u8Vector The interrupt / exception vector number.
2118	* @param fFlags The flags.
2119	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2120	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2121	*/
2122	DECL_NO_INLINE(static, VBOXSTRICTRC)
2123	iemRaiseXcptOrInt(PIEMCPU pIemCpu,
2124	uint8_t cbInstr,
2125	uint8_t u8Vector,
2126	uint32_t fFlags,
2127	uint16_t uErr,
2128	uint64_t uCr2)
2129	{
2130	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2131
2132	/*
2133	* Do recursion accounting.
2134	*/
2135	uint8_t const uPrevXcpt = pIemCpu->uCurXcpt;
2136	uint32_t const fPrevXcpt = pIemCpu->fCurXcpt;
2137	if (pIemCpu->cXcptRecursions == 0)
2138	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
2139	u8Vector, pCtx->cs, pCtx->rip, cbInstr, fFlags, uErr, uCr2));
2140	else
2141	{
2142	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
2143	u8Vector, pCtx->cs, pCtx->rip, cbInstr, fFlags, uErr, uCr2, pIemCpu->uCurXcpt, pIemCpu->cXcptRecursions + 1, fPrevXcpt));
2144
2145	/** @todo double and tripple faults. */
2146	AssertReturn(pIemCpu->cXcptRecursions < 3, VERR_IEM_ASPECT_NOT_IMPLEMENTED);
2147
2148	/** @todo set X86_TRAP_ERR_EXTERNAL when appropriate.
2149	if (fPrevXcpt & IEM_XCPT_FLAGS_T_EXT_INT)
2150	{
2151	....
2152	} */
2153	}
2154	pIemCpu->cXcptRecursions++;
2155	pIemCpu->uCurXcpt = u8Vector;
2156	pIemCpu->fCurXcpt = fFlags;
2157
2158	/*
2159	* Extensive logging.
2160	*/
2161	#ifdef LOG_ENABLED
2162	if (LogIs3Enabled())
2163	{
2164	PVM pVM = IEMCPU_TO_VM(pIemCpu);
2165	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
2166	char szRegs[4096];
2167	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
2168	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
2169	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
2170	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
2171	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
2172	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
2173	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
2174	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
2175	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
2176	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
2177	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
2178	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
2179	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
2180	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
2181	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
2182	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
2183	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
2184	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
2185	" efer=%016VR{efer}\n"
2186	" pat=%016VR{pat}\n"
2187	" sf_mask=%016VR{sf_mask}\n"
2188	"krnl_gs_base=%016VR{krnl_gs_base}\n"
2189	" lstar=%016VR{lstar}\n"
2190	" star=%016VR{star} cstar=%016VR{cstar}\n"
2191	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
2192	);
2193
2194	char szInstr[256];
2195	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
2196	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
2197	szInstr, sizeof(szInstr), NULL);
2198	Log3(("%s%s\n", szRegs, szInstr));
2199	}
2200	#endif /* LOG_ENABLED */
2201
2202	/*
2203	* Call the mode specific worker function.
2204	*/
2205	VBOXSTRICTRC rcStrict;
2206	if (!(pCtx->cr0 & X86_CR0_PE))
2207	rcStrict = iemRaiseXcptOrIntInRealMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2208	else if (pCtx->msrEFER & MSR_K6_EFER_LMA)
2209	rcStrict = iemRaiseXcptOrIntInLongMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2210	else if (!pCtx->eflags.Bits.u1VM)
2211	rcStrict = iemRaiseXcptOrIntInProtMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2212	else
2213	rcStrict = iemRaiseXcptOrIntInV8086Mode(pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2214
2215	/*
2216	* Unwind.
2217	*/
2218	pIemCpu->cXcptRecursions--;
2219	pIemCpu->uCurXcpt = uPrevXcpt;
2220	pIemCpu->fCurXcpt = fPrevXcpt;
2221	LogFlow(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv\n",
2222	VBOXSTRICTRC_VAL(rcStrict), u8Vector, pCtx->cs, pCtx->rip, pCtx->ss, pCtx->esp));
2223	return rcStrict;
2224	}
2225
2226
2227	/** \#DE - 00. */
2228	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDivideError(PIEMCPU pIemCpu)
2229	{
2230	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2231	}
2232
2233
2234	/** \#DB - 01. */
2235	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDebugException(PIEMCPU pIemCpu)
2236	{
2237	/** @todo set/clear RF. */
2238	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2239	}
2240
2241
2242	/** \#UD - 06. */
2243	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseUndefinedOpcode(PIEMCPU pIemCpu)
2244	{
2245	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2246	}
2247
2248
2249	/** \#NM - 07. */
2250	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDeviceNotAvailable(PIEMCPU pIemCpu)
2251	{
2252	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2253	}
2254
2255
2256	#ifdef SOME_UNUSED_FUNCTION
2257	/** \#TS(err) - 0a. */
2258	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultWithErr(PIEMCPU pIemCpu, uint16_t uErr)
2259	{
2260	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2261	}
2262	#endif
2263
2264
2265	/** \#TS(tr) - 0a. */
2266	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultCurrentTSS(PIEMCPU pIemCpu)
2267	{
2268	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2269	pIemCpu->CTX_SUFF(pCtx)->tr, 0);
2270	}
2271
2272
2273	/** \#NP(err) - 0b. */
2274	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr)
2275	{
2276	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2277	}
2278
2279
2280	/** \#NP(seg) - 0b. */
2281	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySegReg(PIEMCPU pIemCpu, uint32_t iSegReg)
2282	{
2283	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2284	iemSRegFetchU16(pIemCpu, iSegReg) & ~X86_SEL_RPL, 0);
2285	}
2286
2287
2288	/** \#NP(sel) - 0b. */
2289	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel)
2290	{
2291	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2292	uSel & ~X86_SEL_RPL, 0);
2293	}
2294
2295
2296	/** \#SS(seg) - 0c. */
2297	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel)
2298	{
2299	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2300	uSel & ~X86_SEL_RPL, 0);
2301	}
2302
2303
2304	/** \#GP(n) - 0d. */
2305	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFault(PIEMCPU pIemCpu, uint16_t uErr)
2306	{
2307	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2308	}
2309
2310
2311	/** \#GP(0) - 0d. */
2312	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFault0(PIEMCPU pIemCpu)
2313	{
2314	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2315	}
2316
2317
2318	/** \#GP(sel) - 0d. */
2319	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFaultBySelector(PIEMCPU pIemCpu, RTSEL Sel)
2320	{
2321	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2322	Sel & ~X86_SEL_RPL, 0);
2323	}
2324
2325
2326	/** \#GP(0) - 0d. */
2327	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseNotCanonical(PIEMCPU pIemCpu)
2328	{
2329	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2330	}
2331
2332
2333	/** \#GP(sel) - 0d. */
2334	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorBounds(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess)
2335	{
2336	NOREF(iSegReg); NOREF(fAccess);
2337	return iemRaiseXcptOrInt(pIemCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
2338	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2339	}
2340
2341
2342	/** \#GP(sel) - 0d. */
2343	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorBoundsBySelector(PIEMCPU pIemCpu, RTSEL Sel)
2344	{
2345	NOREF(Sel);
2346	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2347	}
2348
2349
2350	/** \#GP(sel) - 0d. */
2351	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorInvalidAccess(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess)
2352	{
2353	NOREF(iSegReg); NOREF(fAccess);
2354	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2355	}
2356
2357
2358	/** \#PF(n) - 0e. */
2359	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaisePageFault(PIEMCPU pIemCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
2360	{
2361	uint16_t uErr;
2362	switch (rc)
2363	{
2364	case VERR_PAGE_NOT_PRESENT:
2365	case VERR_PAGE_TABLE_NOT_PRESENT:
2366	case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
2367	case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
2368	uErr = 0;
2369	break;
2370
2371	default:
2372	AssertMsgFailed(("%Rrc\n", rc));
2373	case VERR_ACCESS_DENIED:
2374	uErr = X86_TRAP_PF_P;
2375	break;
2376
2377	/** @todo reserved */
2378	}
2379
2380	if (pIemCpu->uCpl == 3)
2381	uErr \|= X86_TRAP_PF_US;
2382
2383	if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
2384	&& ( (pIemCpu->CTX_SUFF(pCtx)->cr4 & X86_CR4_PAE)
2385	&& (pIemCpu->CTX_SUFF(pCtx)->msrEFER & MSR_K6_EFER_NXE) ) )
2386	uErr \|= X86_TRAP_PF_ID;
2387
2388	/* Note! RW access callers reporting a WRITE protection fault, will clear
2389	the READ flag before calling. So, read-modify-write accesses (RW)
2390	can safely be reported as READ faults. */
2391	if ((fAccess & (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
2392	uErr \|= X86_TRAP_PF_RW;
2393
2394	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR \| IEM_XCPT_FLAGS_CR2,
2395	uErr, GCPtrWhere);
2396	}
2397
2398
2399	/** \#MF(0) - 10. */
2400	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseMathFault(PIEMCPU pIemCpu)
2401	{
2402	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2403	}
2404
2405
2406	/** \#AC(0) - 11. */
2407	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseAlignmentCheckException(PIEMCPU pIemCpu)
2408	{
2409	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2410	}
2411
2412
2413	/**
2414	* Macro for calling iemCImplRaiseDivideError().
2415	*
2416	* This enables us to add/remove arguments and force different levels of
2417	* inlining as we wish.
2418	*
2419	* @return Strict VBox status code.
2420	*/
2421	#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
2422	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
2423	{
2424	NOREF(cbInstr);
2425	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2426	}
2427
2428
2429	/**
2430	* Macro for calling iemCImplRaiseInvalidLockPrefix().
2431	*
2432	* This enables us to add/remove arguments and force different levels of
2433	* inlining as we wish.
2434	*
2435	* @return Strict VBox status code.
2436	*/
2437	#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
2438	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
2439	{
2440	NOREF(cbInstr);
2441	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2442	}
2443
2444
2445	/**
2446	* Macro for calling iemCImplRaiseInvalidOpcode().
2447	*
2448	* This enables us to add/remove arguments and force different levels of
2449	* inlining as we wish.
2450	*
2451	* @return Strict VBox status code.
2452	*/
2453	#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
2454	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
2455	{
2456	NOREF(cbInstr);
2457	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2458	}
2459
2460
2461	/** @} */
2462
2463
2464	/*
2465	*
2466	* Helpers routines.
2467	* Helpers routines.
2468	* Helpers routines.
2469	*
2470	*/
2471
2472	/**
2473	* Recalculates the effective operand size.
2474	*
2475	* @param pIemCpu The IEM state.
2476	*/
2477	static void iemRecalEffOpSize(PIEMCPU pIemCpu)
2478	{
2479	switch (pIemCpu->enmCpuMode)
2480	{
2481	case IEMMODE_16BIT:
2482	pIemCpu->enmEffOpSize = pIemCpu->fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
2483	break;
2484	case IEMMODE_32BIT:
2485	pIemCpu->enmEffOpSize = pIemCpu->fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
2486	break;
2487	case IEMMODE_64BIT:
2488	switch (pIemCpu->fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
2489	{
2490	case 0:
2491	pIemCpu->enmEffOpSize = pIemCpu->enmDefOpSize;
2492	break;
2493	case IEM_OP_PRF_SIZE_OP:
2494	pIemCpu->enmEffOpSize = IEMMODE_16BIT;
2495	break;
2496	case IEM_OP_PRF_SIZE_REX_W:
2497	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
2498	pIemCpu->enmEffOpSize = IEMMODE_64BIT;
2499	break;
2500	}
2501	break;
2502	default:
2503	AssertFailed();
2504	}
2505	}
2506
2507
2508	/**
2509	* Sets the default operand size to 64-bit and recalculates the effective
2510	* operand size.
2511	*
2512	* @param pIemCpu The IEM state.
2513	*/
2514	static void iemRecalEffOpSize64Default(PIEMCPU pIemCpu)
2515	{
2516	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2517	pIemCpu->enmDefOpSize = IEMMODE_64BIT;
2518	if ((pIemCpu->fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
2519	pIemCpu->enmEffOpSize = IEMMODE_64BIT;
2520	else
2521	pIemCpu->enmEffOpSize = IEMMODE_16BIT;
2522	}
2523
2524
2525	/*
2526	*
2527	* Common opcode decoders.
2528	* Common opcode decoders.
2529	* Common opcode decoders.
2530	*
2531	*/
2532	#include <iprt/mem.h>
2533
2534	/**
2535	* Used to add extra details about a stub case.
2536	* @param pIemCpu The IEM per CPU state.
2537	*/
2538	static void iemOpStubMsg2(PIEMCPU pIemCpu)
2539	{
2540	PVM pVM = IEMCPU_TO_VM(pIemCpu);
2541	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
2542	char szRegs[4096];
2543	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
2544	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
2545	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
2546	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
2547	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
2548	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
2549	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
2550	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
2551	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
2552	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
2553	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
2554	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
2555	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
2556	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
2557	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
2558	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
2559	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
2560	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
2561	" efer=%016VR{efer}\n"
2562	" pat=%016VR{pat}\n"
2563	" sf_mask=%016VR{sf_mask}\n"
2564	"krnl_gs_base=%016VR{krnl_gs_base}\n"
2565	" lstar=%016VR{lstar}\n"
2566	" star=%016VR{star} cstar=%016VR{cstar}\n"
2567	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
2568	);
2569
2570	char szInstr[256];
2571	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
2572	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
2573	szInstr, sizeof(szInstr), NULL);
2574
2575	RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
2576	}
2577
2578
2579	/** Stubs an opcode. */
2580	#define FNIEMOP_STUB(a_Name) \
2581	FNIEMOP_DEF(a_Name) \
2582	{ \
2583	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
2584	iemOpStubMsg2(pIemCpu); \
2585	RTAssertPanic(); \
2586	return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
2587	} \
2588	typedef int ignore_semicolon
2589
2590	/** Stubs an opcode. */
2591	#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
2592	FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2593	{ \
2594	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
2595	iemOpStubMsg2(pIemCpu); \
2596	RTAssertPanic(); \
2597	NOREF(a_Name0); \
2598	return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
2599	} \
2600	typedef int ignore_semicolon
2601
2602
2603
2604	/** @name Register Access.
2605	* @{
2606	*/
2607
2608	/**
2609	* Gets a reference (pointer) to the specified hidden segment register.
2610	*
2611	* @returns Hidden register reference.
2612	* @param pIemCpu The per CPU data.
2613	* @param iSegReg The segment register.
2614	*/
2615	static PCPUMSELREGHID iemSRegGetHid(PIEMCPU pIemCpu, uint8_t iSegReg)
2616	{
2617	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2618	switch (iSegReg)
2619	{
2620	case X86_SREG_ES: return &pCtx->esHid;
2621	case X86_SREG_CS: return &pCtx->csHid;
2622	case X86_SREG_SS: return &pCtx->ssHid;
2623	case X86_SREG_DS: return &pCtx->dsHid;
2624	case X86_SREG_FS: return &pCtx->fsHid;
2625	case X86_SREG_GS: return &pCtx->gsHid;
2626	}
2627	AssertFailedReturn(NULL);
2628	}
2629
2630
2631	/**
2632	* Gets a reference (pointer) to the specified segment register (the selector
2633	* value).
2634	*
2635	* @returns Pointer to the selector variable.
2636	* @param pIemCpu The per CPU data.
2637	* @param iSegReg The segment register.
2638	*/
2639	static uint16_t *iemSRegRef(PIEMCPU pIemCpu, uint8_t iSegReg)
2640	{
2641	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2642	switch (iSegReg)
2643	{
2644	case X86_SREG_ES: return &pCtx->es;
2645	case X86_SREG_CS: return &pCtx->cs;
2646	case X86_SREG_SS: return &pCtx->ss;
2647	case X86_SREG_DS: return &pCtx->ds;
2648	case X86_SREG_FS: return &pCtx->fs;
2649	case X86_SREG_GS: return &pCtx->gs;
2650	}
2651	AssertFailedReturn(NULL);
2652	}
2653
2654
2655	/**
2656	* Fetches the selector value of a segment register.
2657	*
2658	* @returns The selector value.
2659	* @param pIemCpu The per CPU data.
2660	* @param iSegReg The segment register.
2661	*/
2662	static uint16_t iemSRegFetchU16(PIEMCPU pIemCpu, uint8_t iSegReg)
2663	{
2664	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2665	switch (iSegReg)
2666	{
2667	case X86_SREG_ES: return pCtx->es;
2668	case X86_SREG_CS: return pCtx->cs;
2669	case X86_SREG_SS: return pCtx->ss;
2670	case X86_SREG_DS: return pCtx->ds;
2671	case X86_SREG_FS: return pCtx->fs;
2672	case X86_SREG_GS: return pCtx->gs;
2673	}
2674	AssertFailedReturn(0xffff);
2675	}
2676
2677
2678	/**
2679	* Gets a reference (pointer) to the specified general register.
2680	*
2681	* @returns Register reference.
2682	* @param pIemCpu The per CPU data.
2683	* @param iReg The general register.
2684	*/
2685	static void *iemGRegRef(PIEMCPU pIemCpu, uint8_t iReg)
2686	{
2687	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2688	switch (iReg)
2689	{
2690	case X86_GREG_xAX: return &pCtx->rax;
2691	case X86_GREG_xCX: return &pCtx->rcx;
2692	case X86_GREG_xDX: return &pCtx->rdx;
2693	case X86_GREG_xBX: return &pCtx->rbx;
2694	case X86_GREG_xSP: return &pCtx->rsp;
2695	case X86_GREG_xBP: return &pCtx->rbp;
2696	case X86_GREG_xSI: return &pCtx->rsi;
2697	case X86_GREG_xDI: return &pCtx->rdi;
2698	case X86_GREG_x8: return &pCtx->r8;
2699	case X86_GREG_x9: return &pCtx->r9;
2700	case X86_GREG_x10: return &pCtx->r10;
2701	case X86_GREG_x11: return &pCtx->r11;
2702	case X86_GREG_x12: return &pCtx->r12;
2703	case X86_GREG_x13: return &pCtx->r13;
2704	case X86_GREG_x14: return &pCtx->r14;
2705	case X86_GREG_x15: return &pCtx->r15;
2706	}
2707	AssertFailedReturn(NULL);
2708	}
2709
2710
2711	/**
2712	* Gets a reference (pointer) to the specified 8-bit general register.
2713	*
2714	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
2715	*
2716	* @returns Register reference.
2717	* @param pIemCpu The per CPU data.
2718	* @param iReg The register.
2719	*/
2720	static uint8_t *iemGRegRefU8(PIEMCPU pIemCpu, uint8_t iReg)
2721	{
2722	if (pIemCpu->fPrefixes & IEM_OP_PRF_REX)
2723	return (uint8_t *)iemGRegRef(pIemCpu, iReg);
2724
2725	uint8_t pu8Reg = (uint8_t )iemGRegRef(pIemCpu, iReg & 3);
2726	if (iReg >= 4)
2727	pu8Reg++;
2728	return pu8Reg;
2729	}
2730
2731
2732	/**
2733	* Fetches the value of a 8-bit general register.
2734	*
2735	* @returns The register value.
2736	* @param pIemCpu The per CPU data.
2737	* @param iReg The register.
2738	*/
2739	static uint8_t iemGRegFetchU8(PIEMCPU pIemCpu, uint8_t iReg)
2740	{
2741	uint8_t const *pbSrc = iemGRegRefU8(pIemCpu, iReg);
2742	return *pbSrc;
2743	}
2744
2745
2746	/**
2747	* Fetches the value of a 16-bit general register.
2748	*
2749	* @returns The register value.
2750	* @param pIemCpu The per CPU data.
2751	* @param iReg The register.
2752	*/
2753	static uint16_t iemGRegFetchU16(PIEMCPU pIemCpu, uint8_t iReg)
2754	{
2755	return (uint16_t )iemGRegRef(pIemCpu, iReg);
2756	}
2757
2758
2759	/**
2760	* Fetches the value of a 32-bit general register.
2761	*
2762	* @returns The register value.
2763	* @param pIemCpu The per CPU data.
2764	* @param iReg The register.
2765	*/
2766	static uint32_t iemGRegFetchU32(PIEMCPU pIemCpu, uint8_t iReg)
2767	{
2768	return (uint32_t )iemGRegRef(pIemCpu, iReg);
2769	}
2770
2771
2772	/**
2773	* Fetches the value of a 64-bit general register.
2774	*
2775	* @returns The register value.
2776	* @param pIemCpu The per CPU data.
2777	* @param iReg The register.
2778	*/
2779	static uint64_t iemGRegFetchU64(PIEMCPU pIemCpu, uint8_t iReg)
2780	{
2781	return (uint64_t )iemGRegRef(pIemCpu, iReg);
2782	}
2783
2784
2785	/**
2786	* Is the FPU state in FXSAVE format or not.
2787	*
2788	* @returns true if it is, false if it's in FNSAVE.
2789	* @param pVCpu The virtual CPU handle.
2790	*/
2791	DECLINLINE(bool) iemFRegIsFxSaveFormat(PIEMCPU pIemCpu)
2792	{
2793	#ifdef RT_ARCH_AMD64
2794	NOREF(pIemCpu);
2795	return true;
2796	#else
2797	NOREF(pIemCpu); /// @todo return pVCpu->pVMR3->cpum.s.CPUFeatures.edx.u1FXSR;
2798	return true;
2799	#endif
2800	}
2801
2802
2803	/**
2804	* Gets the FPU status word.
2805	*
2806	* @returns FPU status word
2807	* @param pIemCpu The per CPU data.
2808	*/
2809	static uint16_t iemFRegFetchFsw(PIEMCPU pIemCpu)
2810	{
2811	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2812	uint16_t u16Fsw;
2813	if (iemFRegIsFxSaveFormat(pIemCpu))
2814	u16Fsw = pCtx->fpu.FSW;
2815	else
2816	{
2817	PX86FPUSTATE pFpu = (PX86FPUSTATE)&pCtx->fpu;
2818	u16Fsw = pFpu->FSW;
2819	}
2820	return u16Fsw;
2821	}
2822
2823	/**
2824	* Adds a 8-bit signed jump offset to RIP/EIP/IP.
2825	*
2826	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2827	* segment limit.
2828	*
2829	* @param pIemCpu The per CPU data.
2830	* @param offNextInstr The offset of the next instruction.
2831	*/
2832	static VBOXSTRICTRC iemRegRipRelativeJumpS8(PIEMCPU pIemCpu, int8_t offNextInstr)
2833	{
2834	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2835	switch (pIemCpu->enmEffOpSize)
2836	{
2837	case IEMMODE_16BIT:
2838	{
2839	uint16_t uNewIp = pCtx->ip + offNextInstr + pIemCpu->offOpcode;
2840	if ( uNewIp > pCtx->csHid.u32Limit
2841	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2842	return iemRaiseGeneralProtectionFault0(pIemCpu);
2843	pCtx->rip = uNewIp;
2844	break;
2845	}
2846
2847	case IEMMODE_32BIT:
2848	{
2849	Assert(pCtx->rip <= UINT32_MAX);
2850	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2851
2852	uint32_t uNewEip = pCtx->eip + offNextInstr + pIemCpu->offOpcode;
2853	if (uNewEip > pCtx->csHid.u32Limit)
2854	return iemRaiseGeneralProtectionFault0(pIemCpu);
2855	pCtx->rip = uNewEip;
2856	break;
2857	}
2858
2859	case IEMMODE_64BIT:
2860	{
2861	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2862
2863	uint64_t uNewRip = pCtx->rip + offNextInstr + pIemCpu->offOpcode;
2864	if (!IEM_IS_CANONICAL(uNewRip))
2865	return iemRaiseGeneralProtectionFault0(pIemCpu);
2866	pCtx->rip = uNewRip;
2867	break;
2868	}
2869
2870	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2871	}
2872
2873	return VINF_SUCCESS;
2874	}
2875
2876
2877	/**
2878	* Adds a 16-bit signed jump offset to RIP/EIP/IP.
2879	*
2880	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2881	* segment limit.
2882	*
2883	* @returns Strict VBox status code.
2884	* @param pIemCpu The per CPU data.
2885	* @param offNextInstr The offset of the next instruction.
2886	*/
2887	static VBOXSTRICTRC iemRegRipRelativeJumpS16(PIEMCPU pIemCpu, int16_t offNextInstr)
2888	{
2889	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2890	Assert(pIemCpu->enmEffOpSize == IEMMODE_16BIT);
2891
2892	uint16_t uNewIp = pCtx->ip + offNextInstr + pIemCpu->offOpcode;
2893	if ( uNewIp > pCtx->csHid.u32Limit
2894	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2895	return iemRaiseGeneralProtectionFault0(pIemCpu);
2896	/** @todo Test 16-bit jump in 64-bit mode. */
2897	pCtx->rip = uNewIp;
2898
2899	return VINF_SUCCESS;
2900	}
2901
2902
2903	/**
2904	* Adds a 32-bit signed jump offset to RIP/EIP/IP.
2905	*
2906	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2907	* segment limit.
2908	*
2909	* @returns Strict VBox status code.
2910	* @param pIemCpu The per CPU data.
2911	* @param offNextInstr The offset of the next instruction.
2912	*/
2913	static VBOXSTRICTRC iemRegRipRelativeJumpS32(PIEMCPU pIemCpu, int32_t offNextInstr)
2914	{
2915	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2916	Assert(pIemCpu->enmEffOpSize != IEMMODE_16BIT);
2917
2918	if (pIemCpu->enmEffOpSize == IEMMODE_32BIT)
2919	{
2920	Assert(pCtx->rip <= UINT32_MAX); Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2921
2922	uint32_t uNewEip = pCtx->eip + offNextInstr + pIemCpu->offOpcode;
2923	if (uNewEip > pCtx->csHid.u32Limit)
2924	return iemRaiseGeneralProtectionFault0(pIemCpu);
2925	pCtx->rip = uNewEip;
2926	}
2927	else
2928	{
2929	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2930
2931	uint64_t uNewRip = pCtx->rip + offNextInstr + pIemCpu->offOpcode;
2932	if (!IEM_IS_CANONICAL(uNewRip))
2933	return iemRaiseGeneralProtectionFault0(pIemCpu);
2934	pCtx->rip = uNewRip;
2935	}
2936	return VINF_SUCCESS;
2937	}
2938
2939
2940	/**
2941	* Performs a near jump to the specified address.
2942	*
2943	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2944	* segment limit.
2945	*
2946	* @param pIemCpu The per CPU data.
2947	* @param uNewRip The new RIP value.
2948	*/
2949	static VBOXSTRICTRC iemRegRipJump(PIEMCPU pIemCpu, uint64_t uNewRip)
2950	{
2951	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2952	switch (pIemCpu->enmEffOpSize)
2953	{
2954	case IEMMODE_16BIT:
2955	{
2956	Assert(uNewRip <= UINT16_MAX);
2957	if ( uNewRip > pCtx->csHid.u32Limit
2958	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2959	return iemRaiseGeneralProtectionFault0(pIemCpu);
2960	/** @todo Test 16-bit jump in 64-bit mode. */
2961	pCtx->rip = uNewRip;
2962	break;
2963	}
2964
2965	case IEMMODE_32BIT:
2966	{
2967	Assert(uNewRip <= UINT32_MAX);
2968	Assert(pCtx->rip <= UINT32_MAX);
2969	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2970
2971	if (uNewRip > pCtx->csHid.u32Limit)
2972	return iemRaiseGeneralProtectionFault0(pIemCpu);
2973	pCtx->rip = uNewRip;
2974	break;
2975	}
2976
2977	case IEMMODE_64BIT:
2978	{
2979	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2980
2981	if (!IEM_IS_CANONICAL(uNewRip))
2982	return iemRaiseGeneralProtectionFault0(pIemCpu);
2983	pCtx->rip = uNewRip;
2984	break;
2985	}
2986
2987	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2988	}
2989
2990	return VINF_SUCCESS;
2991	}
2992
2993
2994	/**
2995	* Get the address of the top of the stack.
2996	*
2997	* @param pCtx The CPU context which SP/ESP/RSP should be
2998	* read.
2999	*/
3000	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCCPUMCTX pCtx)
3001	{
3002	if (pCtx->ssHid.Attr.n.u1Long)
3003	return pCtx->rsp;
3004	if (pCtx->ssHid.Attr.n.u1DefBig)
3005	return pCtx->esp;
3006	return pCtx->sp;
3007	}
3008
3009
3010	/**
3011	* Updates the RIP/EIP/IP to point to the next instruction.
3012	*
3013	* @param pIemCpu The per CPU data.
3014	* @param cbInstr The number of bytes to add.
3015	*/
3016	static void iemRegAddToRip(PIEMCPU pIemCpu, uint8_t cbInstr)
3017	{
3018	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3019	switch (pIemCpu->enmCpuMode)
3020	{
3021	case IEMMODE_16BIT:
3022	Assert(pCtx->rip <= UINT16_MAX);
3023	pCtx->eip += cbInstr;
3024	pCtx->eip &= UINT32_C(0xffff);
3025	break;
3026
3027	case IEMMODE_32BIT:
3028	pCtx->eip += cbInstr;
3029	Assert(pCtx->rip <= UINT32_MAX);
3030	break;
3031
3032	case IEMMODE_64BIT:
3033	pCtx->rip += cbInstr;
3034	break;
3035	default: AssertFailed();
3036	}
3037	}
3038
3039
3040	/**
3041	* Updates the RIP/EIP/IP to point to the next instruction.
3042	*
3043	* @param pIemCpu The per CPU data.
3044	*/
3045	static void iemRegUpdateRip(PIEMCPU pIemCpu)
3046	{
3047	return iemRegAddToRip(pIemCpu, pIemCpu->offOpcode);
3048	}
3049
3050
3051	/**
3052	* Adds to the stack pointer.
3053	*
3054	* @param pCtx The CPU context which SP/ESP/RSP should be
3055	* updated.
3056	* @param cbToAdd The number of bytes to add.
3057	*/
3058	DECLINLINE(void) iemRegAddToRsp(PCPUMCTX pCtx, uint8_t cbToAdd)
3059	{
3060	if (pCtx->ssHid.Attr.n.u1Long)
3061	pCtx->rsp += cbToAdd;
3062	else if (pCtx->ssHid.Attr.n.u1DefBig)
3063	pCtx->esp += cbToAdd;
3064	else
3065	pCtx->sp += cbToAdd;
3066	}
3067
3068
3069	/**
3070	* Subtracts from the stack pointer.
3071	*
3072	* @param pCtx The CPU context which SP/ESP/RSP should be
3073	* updated.
3074	* @param cbToSub The number of bytes to subtract.
3075	*/
3076	DECLINLINE(void) iemRegSubFromRsp(PCPUMCTX pCtx, uint8_t cbToSub)
3077	{
3078	if (pCtx->ssHid.Attr.n.u1Long)
3079	pCtx->rsp -= cbToSub;
3080	else if (pCtx->ssHid.Attr.n.u1DefBig)
3081	pCtx->esp -= cbToSub;
3082	else
3083	pCtx->sp -= cbToSub;
3084	}
3085
3086
3087	/**
3088	* Adds to the temporary stack pointer.
3089	*
3090	* @param pTmpRsp The temporary SP/ESP/RSP to update.
3091	* @param cbToAdd The number of bytes to add.
3092	* @param pCtx Where to get the current stack mode.
3093	*/
3094	DECLINLINE(void) iemRegAddToRspEx(PRTUINT64U pTmpRsp, uint8_t cbToAdd, PCCPUMCTX pCtx)
3095	{
3096	if (pCtx->ssHid.Attr.n.u1Long)
3097	pTmpRsp->u += cbToAdd;
3098	else if (pCtx->ssHid.Attr.n.u1DefBig)
3099	pTmpRsp->DWords.dw0 += cbToAdd;
3100	else
3101	pTmpRsp->Words.w0 += cbToAdd;
3102	}
3103
3104
3105	/**
3106	* Subtracts from the temporary stack pointer.
3107	*
3108	* @param pTmpRsp The temporary SP/ESP/RSP to update.
3109	* @param cbToSub The number of bytes to subtract.
3110	* @param pCtx Where to get the current stack mode.
3111	*/
3112	DECLINLINE(void) iemRegSubFromRspEx(PRTUINT64U pTmpRsp, uint8_t cbToSub, PCCPUMCTX pCtx)
3113	{
3114	if (pCtx->ssHid.Attr.n.u1Long)
3115	pTmpRsp->u -= cbToSub;
3116	else if (pCtx->ssHid.Attr.n.u1DefBig)
3117	pTmpRsp->DWords.dw0 -= cbToSub;
3118	else
3119	pTmpRsp->Words.w0 -= cbToSub;
3120	}
3121
3122
3123	/**
3124	* Calculates the effective stack address for a push of the specified size as
3125	* well as the new RSP value (upper bits may be masked).
3126	*
3127	* @returns Effective stack addressf for the push.
3128	* @param pCtx Where to get the current stack mode.
3129	* @param cbItem The size of the stack item to pop.
3130	* @param puNewRsp Where to return the new RSP value.
3131	*/
3132	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCCPUMCTX pCtx, uint8_t cbItem, uint64_t *puNewRsp)
3133	{
3134	RTUINT64U uTmpRsp;
3135	RTGCPTR GCPtrTop;
3136	uTmpRsp.u = pCtx->rsp;
3137
3138	if (pCtx->ssHid.Attr.n.u1Long)
3139	GCPtrTop = uTmpRsp.u -= cbItem;
3140	else if (pCtx->ssHid.Attr.n.u1DefBig)
3141	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
3142	else
3143	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
3144	*puNewRsp = uTmpRsp.u;
3145	return GCPtrTop;
3146	}
3147
3148
3149	/**
3150	* Gets the current stack pointer and calculates the value after a pop of the
3151	* specified size.
3152	*
3153	* @returns Current stack pointer.
3154	* @param pCtx Where to get the current stack mode.
3155	* @param cbItem The size of the stack item to pop.
3156	* @param puNewRsp Where to return the new RSP value.
3157	*/
3158	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCCPUMCTX pCtx, uint8_t cbItem, uint64_t *puNewRsp)
3159	{
3160	RTUINT64U uTmpRsp;
3161	RTGCPTR GCPtrTop;
3162	uTmpRsp.u = pCtx->rsp;
3163
3164	if (pCtx->ssHid.Attr.n.u1Long)
3165	{
3166	GCPtrTop = uTmpRsp.u;
3167	uTmpRsp.u += cbItem;
3168	}
3169	else if (pCtx->ssHid.Attr.n.u1DefBig)
3170	{
3171	GCPtrTop = uTmpRsp.DWords.dw0;
3172	uTmpRsp.DWords.dw0 += cbItem;
3173	}
3174	else
3175	{
3176	GCPtrTop = uTmpRsp.Words.w0;
3177	uTmpRsp.Words.w0 += cbItem;
3178	}
3179	*puNewRsp = uTmpRsp.u;
3180	return GCPtrTop;
3181	}
3182
3183
3184	/**
3185	* Calculates the effective stack address for a push of the specified size as
3186	* well as the new temporary RSP value (upper bits may be masked).
3187	*
3188	* @returns Effective stack addressf for the push.
3189	* @param pTmpRsp The temporary stack pointer. This is updated.
3190	* @param cbItem The size of the stack item to pop.
3191	* @param puNewRsp Where to return the new RSP value.
3192	*/
3193	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PRTUINT64U pTmpRsp, uint8_t cbItem, PCCPUMCTX pCtx)
3194	{
3195	RTGCPTR GCPtrTop;
3196
3197	if (pCtx->ssHid.Attr.n.u1Long)
3198	GCPtrTop = pTmpRsp->u -= cbItem;
3199	else if (pCtx->ssHid.Attr.n.u1DefBig)
3200	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
3201	else
3202	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
3203	return GCPtrTop;
3204	}
3205
3206
3207	/**
3208	* Gets the effective stack address for a pop of the specified size and
3209	* calculates and updates the temporary RSP.
3210	*
3211	* @returns Current stack pointer.
3212	* @param pTmpRsp The temporary stack pointer. This is updated.
3213	* @param pCtx Where to get the current stack mode.
3214	* @param cbItem The size of the stack item to pop.
3215	*/
3216	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PRTUINT64U pTmpRsp, uint8_t cbItem, PCCPUMCTX pCtx)
3217	{
3218	RTGCPTR GCPtrTop;
3219	if (pCtx->ssHid.Attr.n.u1Long)
3220	{
3221	GCPtrTop = pTmpRsp->u;
3222	pTmpRsp->u += cbItem;
3223	}
3224	else if (pCtx->ssHid.Attr.n.u1DefBig)
3225	{
3226	GCPtrTop = pTmpRsp->DWords.dw0;
3227	pTmpRsp->DWords.dw0 += cbItem;
3228	}
3229	else
3230	{
3231	GCPtrTop = pTmpRsp->Words.w0;
3232	pTmpRsp->Words.w0 += cbItem;
3233	}
3234	return GCPtrTop;
3235	}
3236
3237
3238	/**
3239	* Checks if an Intel CPUID feature bit is set.
3240	*
3241	* @returns true / false.
3242	*
3243	* @param pIemCpu The IEM per CPU data.
3244	* @param fEdx The EDX bit to test, or 0 if ECX.
3245	* @param fEcx The ECX bit to test, or 0 if EDX.
3246	* @remarks Used via IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX,
3247	* IEM_IS_INTEL_CPUID_FEATURE_PRESENT_ECX and others.
3248	*/
3249	static bool iemRegIsIntelCpuIdFeaturePresent(PIEMCPU pIemCpu, uint32_t fEdx, uint32_t fEcx)
3250	{
3251	uint32_t uEax, uEbx, uEcx, uEdx;
3252	CPUMGetGuestCpuId(IEMCPU_TO_VMCPU(pIemCpu), 0x00000001, &uEax, &uEbx, &uEcx, &uEdx);
3253	return (fEcx && (uEcx & fEcx))
3254	\|\| (fEdx && (uEdx & fEdx));
3255	}
3256
3257
3258	/**
3259	* Checks if an AMD CPUID feature bit is set.
3260	*
3261	* @returns true / false.
3262	*
3263	* @param pIemCpu The IEM per CPU data.
3264	* @param fEdx The EDX bit to test, or 0 if ECX.
3265	* @param fEcx The ECX bit to test, or 0 if EDX.
3266	* @remarks Used via IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX,
3267	* IEM_IS_AMD_CPUID_FEATURE_PRESENT_ECX and others.
3268	*/
3269	static bool iemRegIsAmdCpuIdFeaturePresent(PIEMCPU pIemCpu, uint32_t fEdx, uint32_t fEcx)
3270	{
3271	uint32_t uEax, uEbx, uEcx, uEdx;
3272	CPUMGetGuestCpuId(IEMCPU_TO_VMCPU(pIemCpu), 0x80000001, &uEax, &uEbx, &uEcx, &uEdx);
3273	return (fEcx && (uEcx & fEcx))
3274	\|\| (fEdx && (uEdx & fEdx));
3275	}
3276
3277	/** @} */
3278
3279
3280	/** @name FPU access and helpers.
3281	*
3282	* @{
3283	*/
3284
3285
3286	/**
3287	* Hook for preparing to use the host FPU.
3288	*
3289	* This is necessary in ring-0 and raw-mode context.
3290	*
3291	* @param pIemCpu The IEM per CPU data.
3292	*/
3293	DECLINLINE(void) iemFpuPrepareUsage(PIEMCPU pIemCpu)
3294	{
3295	#ifdef IN_RING3
3296	NOREF(pIemCpu);
3297	#else
3298	# error "Implement me"
3299	#endif
3300	}
3301
3302
3303	/**
3304	* Stores a QNaN value into a FPU register.
3305	*
3306	* @param pReg Pointer to the register.
3307	*/
3308	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
3309	{
3310	pReg->au32[0] = UINT32_C(0x00000000);
3311	pReg->au32[1] = UINT32_C(0xc0000000);
3312	pReg->au16[4] = UINT16_C(0xffff);
3313	}
3314
3315
3316	/**
3317	* Updates the FOP, FPU.CS and FPUIP registers.
3318	*
3319	* @param pIemCpu The IEM per CPU data.
3320	* @param pCtx The CPU context.
3321	*/
3322	DECLINLINE(void) iemFpuUpdateOpcodeAndIP(PIEMCPU pIemCpu, PCPUMCTX pCtx)
3323	{
3324	pCtx->fpu.FOP = pIemCpu->abOpcode[pIemCpu->offFpuOpcode]
3325	\| ((uint16_t)(pIemCpu->abOpcode[pIemCpu->offFpuOpcode - 1] & 0x7) << 8);
3326	/** @todo FPU.CS and FPUIP needs to be kept seperately. */
3327	pCtx->fpu.CS = pCtx->cs;
3328	pCtx->fpu.FPUIP = pCtx->rip;
3329	}
3330
3331
3332	/**
3333	* Updates the FPU.DS and FPUDP registers.
3334	*
3335	* @param pIemCpu The IEM per CPU data.
3336	* @param pCtx The CPU context.
3337	* @param iEffSeg The effective segment register.
3338	* @param GCPtrEff The effective address relative to @a iEffSeg.
3339	*/
3340	DECLINLINE(void) iemFpuUpdateDP(PIEMCPU pIemCpu, PCPUMCTX pCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3341	{
3342	RTSEL sel;
3343	switch (iEffSeg)
3344	{
3345	case X86_SREG_DS: sel = pCtx->ds; break;
3346	case X86_SREG_SS: sel = pCtx->ss; break;
3347	case X86_SREG_CS: sel = pCtx->cs; break;
3348	case X86_SREG_ES: sel = pCtx->es; break;
3349	case X86_SREG_FS: sel = pCtx->fs; break;
3350	case X86_SREG_GS: sel = pCtx->gs; break;
3351	default:
3352	AssertMsgFailed(("%d\n", iEffSeg));
3353	sel = pCtx->ds;
3354	}
3355	/** @todo FPU.DS and FPUDP needs to be kept seperately. */
3356	pCtx->fpu.DS = sel;
3357	pCtx->fpu.FPUDP = GCPtrEff;
3358	}
3359
3360
3361	/**
3362	* Rotates the stack registers in the push direction.
3363	*
3364	* @param pCtx The CPU context.
3365	* @remarks This is a complete waste of time, but fxsave stores the registers in
3366	* stack order.
3367	*/
3368	DECLINLINE(void) iemFpuRotateStackPush(PCPUMCTX pCtx)
3369	{
3370	RTFLOAT80U r80Tmp = pCtx->fpu.aRegs[7].r80;
3371	pCtx->fpu.aRegs[7].r80 = pCtx->fpu.aRegs[6].r80;
3372	pCtx->fpu.aRegs[6].r80 = pCtx->fpu.aRegs[5].r80;
3373	pCtx->fpu.aRegs[5].r80 = pCtx->fpu.aRegs[4].r80;
3374	pCtx->fpu.aRegs[4].r80 = pCtx->fpu.aRegs[3].r80;
3375	pCtx->fpu.aRegs[3].r80 = pCtx->fpu.aRegs[2].r80;
3376	pCtx->fpu.aRegs[2].r80 = pCtx->fpu.aRegs[1].r80;
3377	pCtx->fpu.aRegs[1].r80 = pCtx->fpu.aRegs[0].r80;
3378	pCtx->fpu.aRegs[0].r80 = r80Tmp;
3379	}
3380
3381
3382	/**
3383	* Rotates the stack registers in the pop direction.
3384	*
3385	* @param pCtx The CPU context.
3386	* @remarks This is a complete waste of time, but fxsave stores the registers in
3387	* stack order.
3388	*/
3389	DECLINLINE(void) iemFpuRotateStackPop(PCPUMCTX pCtx)
3390	{
3391	RTFLOAT80U r80Tmp = pCtx->fpu.aRegs[0].r80;
3392	pCtx->fpu.aRegs[0].r80 = pCtx->fpu.aRegs[1].r80;
3393	pCtx->fpu.aRegs[1].r80 = pCtx->fpu.aRegs[2].r80;
3394	pCtx->fpu.aRegs[2].r80 = pCtx->fpu.aRegs[3].r80;
3395	pCtx->fpu.aRegs[3].r80 = pCtx->fpu.aRegs[4].r80;
3396	pCtx->fpu.aRegs[4].r80 = pCtx->fpu.aRegs[5].r80;
3397	pCtx->fpu.aRegs[5].r80 = pCtx->fpu.aRegs[6].r80;
3398	pCtx->fpu.aRegs[6].r80 = pCtx->fpu.aRegs[7].r80;
3399	pCtx->fpu.aRegs[7].r80 = r80Tmp;
3400	}
3401
3402
3403	/**
3404	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
3405	*
3406	* @param pIemCpu The IEM per CPU data.
3407	* @param pResult The FPU operation result to push.
3408	* @param pCtx The CPU context.
3409	*/
3410	static void iemFpuMaybePushResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult, PCPUMCTX pCtx)
3411	{
3412	/* Check pending exceptions. */
3413	uint16_t uFSW = pCtx->fpu.FSW;
3414	if ( (pCtx->fpu.FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
3415	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
3416	return;
3417
3418	/* Push it. */
3419	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
3420	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
3421	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_TOP_MASK;
3422	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
3423	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3424	pCtx->fpu.aRegs[7].r80 = pResult->r80Result;
3425
3426	iemFpuRotateStackPush(pCtx);
3427	}
3428
3429
3430	/**
3431	* Stores a result in a FPU register and updates the FSW and FTW.
3432	*
3433	* @param pIemCpu The IEM per CPU data.
3434	* @param pResult The result to store.
3435	* @param iStReg Which FPU register to store it in.
3436	* @param pCtx The CPU context.
3437	*/
3438	static void iemFpuStoreResultOnly(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg, PCPUMCTX pCtx)
3439	{
3440	Assert(iStReg < 8);
3441	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3442	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3443	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_TOP_MASK;
3444	pCtx->fpu.FTW \|= RT_BIT(iReg);
3445	pCtx->fpu.aRegs[iStReg].r80 = pResult->r80Result;
3446	}
3447
3448
3449	/**
3450	* Only updates the FPU status word (FSW) with the result of the current
3451	* instruction.
3452	*
3453	* @param pCtx The CPU context.
3454	* @param u16FSW The FSW output of the current instruction.
3455	*/
3456	static void iemFpuUpdateFSWOnly(PCPUMCTX pCtx, uint16_t u16FSW)
3457	{
3458	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3459	pCtx->fpu.FSW \|= u16FSW & ~X86_FSW_TOP_MASK;
3460	}
3461
3462
3463	/**
3464	* Pops one item off the FPU stack if no pending exception prevents it.
3465	*
3466	* @param pCtx The CPU context.
3467	*/
3468	static void iemFpuMaybePopOne(PCPUMCTX pCtx)
3469	{
3470	/* Check pending exceptions. */
3471	uint16_t uFSW = pCtx->fpu.FSW;
3472	if ( (pCtx->fpu.FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
3473	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
3474	return;
3475
3476	/* TOP--. */
3477	uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
3478	uFSW &= ~X86_FSW_TOP_MASK;
3479	uFSW \|= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3480	pCtx->fpu.FSW = uFSW;
3481
3482	/* Mark the previous ST0 as empty. */
3483	iOldTop >>= X86_FSW_TOP_SHIFT;
3484	pCtx->fpu.FTW &= ~RT_BIT(iOldTop);
3485
3486	/* Rotate the registers. */
3487	iemFpuRotateStackPop(pCtx);
3488	}
3489
3490
3491	/**
3492	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
3493	*
3494	* @param pIemCpu The IEM per CPU data.
3495	* @param pResult The FPU operation result to push.
3496	*/
3497	static void iemFpuPushResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult)
3498	{
3499	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3500	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3501	iemFpuMaybePushResult(pIemCpu, pResult, pCtx);
3502	}
3503
3504
3505	/**
3506	* Pushes a FPU result onto the FPU stack if no pending exception prevents it,
3507	* and sets FPUDP and FPUDS.
3508	*
3509	* @param pIemCpu The IEM per CPU data.
3510	* @param pResult The FPU operation result to push.
3511	* @param iEffSeg The effective segment register.
3512	* @param GCPtrEff The effective address relative to @a iEffSeg.
3513	*/
3514	static void iemFpuPushResultWithMemOp(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3515	{
3516	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3517	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3518	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3519	iemFpuMaybePushResult(pIemCpu, pResult, pCtx);
3520	}
3521
3522
3523	/**
3524	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
3525	* FOP.
3526	*
3527	* @param pIemCpu The IEM per CPU data.
3528	* @param pResult The result to store.
3529	* @param iStReg Which FPU register to store it in.
3530	* @param pCtx The CPU context.
3531	*/
3532	static void iemFpuStoreResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg)
3533	{
3534	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3535	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3536	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3537	}
3538
3539
3540	/**
3541	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
3542	* FOP, and then pops the stack.
3543	*
3544	* @param pIemCpu The IEM per CPU data.
3545	* @param pResult The result to store.
3546	* @param iStReg Which FPU register to store it in.
3547	* @param pCtx The CPU context.
3548	*/
3549	static void iemFpuStoreResultThenPop(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg)
3550	{
3551	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3552	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3553	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3554	iemFpuMaybePopOne(pCtx);
3555	}
3556
3557
3558	/**
3559	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
3560	* FPUDP, and FPUDS.
3561	*
3562	* @param pIemCpu The IEM per CPU data.
3563	* @param pResult The result to store.
3564	* @param iStReg Which FPU register to store it in.
3565	* @param pCtx The CPU context.
3566	* @param iEffSeg The effective memory operand selector register.
3567	* @param GCPtrEff The effective memory operand offset.
3568	*/
3569	static void iemFpuStoreResultWithMemOp(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3570	{
3571	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3572	iemFpuUpdateDP(pIemCpu, pIemCpu->CTX_SUFF(pCtx), iEffSeg, GCPtrEff);
3573	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3574	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3575	}
3576
3577
3578	/**
3579	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
3580	* FPUDP, and FPUDS, and then pops the stack.
3581	*
3582	* @param pIemCpu The IEM per CPU data.
3583	* @param pResult The result to store.
3584	* @param iStReg Which FPU register to store it in.
3585	* @param pCtx The CPU context.
3586	* @param iEffSeg The effective memory operand selector register.
3587	* @param GCPtrEff The effective memory operand offset.
3588	*/
3589	static void iemFpuStoreResultWithMemOpThenPop(PIEMCPU pIemCpu, PIEMFPURESULT pResult,
3590	uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3591	{
3592	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3593	iemFpuUpdateDP(pIemCpu, pIemCpu->CTX_SUFF(pCtx), iEffSeg, GCPtrEff);
3594	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3595	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3596	iemFpuMaybePopOne(pCtx);
3597	}
3598
3599
3600	/**
3601	* Updates the FSW, FOP, FPUIP, and FPUCS.
3602	*
3603	* @param pIemCpu The IEM per CPU data.
3604	* @param u16FSW The FSW from the current instruction.
3605	*/
3606	static void iemFpuUpdateFSW(PIEMCPU pIemCpu, uint16_t u16FSW)
3607	{
3608	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3609	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3610	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3611	}
3612
3613
3614	/**
3615	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
3616	*
3617	* @param pIemCpu The IEM per CPU data.
3618	* @param u16FSW The FSW from the current instruction.
3619	*/
3620	static void iemFpuUpdateFSWThenPop(PIEMCPU pIemCpu, uint16_t u16FSW)
3621	{
3622	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3623	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3624	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3625	iemFpuMaybePopOne(pCtx);
3626	}
3627
3628
3629	/**
3630	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
3631	*
3632	* @param pIemCpu The IEM per CPU data.
3633	* @param u16FSW The FSW from the current instruction.
3634	* @param iEffSeg The effective memory operand selector register.
3635	* @param GCPtrEff The effective memory operand offset.
3636	*/
3637	static void iemFpuUpdateFSWWithMemOp(PIEMCPU pIemCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3638	{
3639	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3640	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3641	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3642	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3643	}
3644
3645
3646	/**
3647	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
3648	*
3649	* @param pIemCpu The IEM per CPU data.
3650	* @param u16FSW The FSW from the current instruction.
3651	* @param iEffSeg The effective memory operand selector register.
3652	* @param GCPtrEff The effective memory operand offset.
3653	*/
3654	static void iemFpuUpdateFSWWithMemOpThenPop(PIEMCPU pIemCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3655	{
3656	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3657	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3658	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3659	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3660	iemFpuMaybePopOne(pCtx);
3661	}
3662
3663
3664	/**
3665	* Worker routine for raising an FPU stack underflow exception.
3666	*
3667	* @param pIemCpu The IEM per CPU data.
3668	* @param iStReg The stack register being accessed.
3669	* @param pCtx The CPU context.
3670	*/
3671	static void iemFpuStackUnderflowOnly(PIEMCPU pIemCpu, uint8_t iStReg, PCPUMCTX pCtx)
3672	{
3673	Assert(iStReg < 8 \|\| iStReg == UINT8_MAX);
3674	if (pCtx->fpu.FCW & X86_FCW_IM)
3675	{
3676	/* Masked underflow. */
3677	pCtx->fpu.FSW &= ~(X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3);
3678	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
3679	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3680	if (iStReg != UINT8_MAX)
3681	{
3682	pCtx->fpu.FTW \|= RT_BIT(iReg);
3683	iemFpuStoreQNan(&pCtx->fpu.aRegs[iStReg].r80);
3684	}
3685	}
3686	else
3687	{
3688	pCtx->fpu.FSW &= ~(X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3);
3689	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3690	}
3691	}
3692
3693
3694	/**
3695	* Raises a FPU stack underflow exception.
3696	*
3697	* @param pIemCpu The IEM per CPU data.
3698	* @param iStReg The destination register that should be loaded
3699	* with QNaN if \#IS is not masked. Specify
3700	* UINT8_MAX if none (like for fcom).
3701	*/
3702	DECL_NO_INLINE(static, void) iemFpuStackUnderflow(PIEMCPU pIemCpu, uint8_t iStReg)
3703	{
3704	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3705	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3706	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3707	}
3708
3709
3710	DECL_NO_INLINE(static, void)
3711	iemFpuStackUnderflowWithMemOp(PIEMCPU pIemCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3712	{
3713	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3714	iemFpuUpdateDP(pIemCpu, pIemCpu->CTX_SUFF(pCtx), iEffSeg, GCPtrEff);
3715	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3716	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3717	}
3718
3719
3720	DECL_NO_INLINE(static, void) iemFpuStackUnderflowThenPop(PIEMCPU pIemCpu, uint8_t iStReg)
3721	{
3722	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3723	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3724	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3725	iemFpuMaybePopOne(pCtx);
3726	}
3727
3728
3729	DECL_NO_INLINE(static, void)
3730	iemFpuStackUnderflowWithMemOpThenPop(PIEMCPU pIemCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3731	{
3732	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3733	iemFpuUpdateDP(pIemCpu, pIemCpu->CTX_SUFF(pCtx), iEffSeg, GCPtrEff);
3734	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3735	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3736	iemFpuMaybePopOne(pCtx);
3737	}
3738
3739
3740	/**
3741	* Worker routine for raising an FPU stack overflow exception on a push.
3742	*
3743	* @param pIemCpu The IEM per CPU data.
3744	* @param pCtx The CPU context.
3745	*/
3746	static void iemFpuStackPushOverflowOnly(PIEMCPU pIemCpu, PCPUMCTX pCtx)
3747	{
3748	if (pCtx->fpu.FCW & X86_FCW_IM)
3749	{
3750	/* Masked overflow. */
3751	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
3752	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3);
3753	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
3754	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
3755	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3756	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3757	iemFpuRotateStackPush(pCtx);
3758	}
3759	else
3760	{
3761	/* Exception pending - don't change TOP or the register stack. */
3762	pCtx->fpu.FSW &= ~(X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3);
3763	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3764	}
3765	}
3766
3767
3768	/**
3769	* Raises a FPU stack overflow exception on a push.
3770	*
3771	* @param pIemCpu The IEM per CPU data.
3772	*/
3773	DECL_NO_INLINE(static, void) iemFpuStackPushOverflow(PIEMCPU pIemCpu)
3774	{
3775	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3776	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3777	iemFpuStackPushOverflowOnly(pIemCpu, pCtx);
3778	}
3779
3780
3781	/**
3782	* Raises a FPU stack overflow exception on a push with a memory operand.
3783	*
3784	* @param pIemCpu The IEM per CPU data.
3785	* @param iEffSeg The effective memory operand selector register.
3786	* @param GCPtrEff The effective memory operand offset.
3787	*/
3788	DECL_NO_INLINE(static, void)
3789	iemFpuStackPushOverflowWithMemOp(PIEMCPU pIemCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3790	{
3791	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3792	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3793	iemFpuUpdateOpcodeAndIP(pIemCpu, pCtx);
3794	iemFpuStackPushOverflowOnly(pIemCpu, pCtx);
3795	}
3796
3797
3798	static int iemFpuStRegNotEmpty(PIEMCPU pIemCpu, uint8_t iStReg)
3799	{
3800	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3801	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3802	if (pCtx->fpu.FTW & RT_BIT(iReg))
3803	return VINF_SUCCESS;
3804	return VERR_NOT_FOUND;
3805	}
3806
3807
3808	static int iemFpuStRegNotEmptyRef(PIEMCPU pIemCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
3809	{
3810	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3811	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3812	if (pCtx->fpu.FTW & RT_BIT(iReg))
3813	{
3814	*ppRef = &pCtx->fpu.aRegs[iStReg].r80;
3815	return VINF_SUCCESS;
3816	}
3817	return VERR_NOT_FOUND;
3818	}
3819
3820
3821	static int iemFpu2StRegsNotEmptyRef(PIEMCPU pIemCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
3822	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
3823	{
3824	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3825	uint16_t iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
3826	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
3827	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
3828	if ((pCtx->fpu.FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
3829	{
3830	*ppRef0 = &pCtx->fpu.aRegs[iStReg0].r80;
3831	*ppRef1 = &pCtx->fpu.aRegs[iStReg1].r80;
3832	return VINF_SUCCESS;
3833	}
3834	return VERR_NOT_FOUND;
3835	}
3836
3837	/** @} */
3838
3839
3840	/** @name Memory access.
3841	*
3842	* @{
3843	*/
3844
3845
3846	/**
3847	* Checks if the given segment can be written to, raise the appropriate
3848	* exception if not.
3849	*
3850	* @returns VBox strict status code.
3851	*
3852	* @param pIemCpu The IEM per CPU data.
3853	* @param pHid Pointer to the hidden register.
3854	* @param iSegReg The register number.
3855	*/
3856	static VBOXSTRICTRC iemMemSegCheckWriteAccessEx(PIEMCPU pIemCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg)
3857	{
3858	if (!pHid->Attr.n.u1Present)
3859	return iemRaiseSelectorNotPresentBySegReg(pIemCpu, iSegReg);
3860
3861	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3862	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3863	&& pIemCpu->enmCpuMode != IEMMODE_64BIT )
3864	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, IEM_ACCESS_DATA_W);
3865
3866	/** @todo DPL/RPL/CPL? */
3867
3868	return VINF_SUCCESS;
3869	}
3870
3871
3872	/**
3873	* Checks if the given segment can be read from, raise the appropriate
3874	* exception if not.
3875	*
3876	* @returns VBox strict status code.
3877	*
3878	* @param pIemCpu The IEM per CPU data.
3879	* @param pHid Pointer to the hidden register.
3880	* @param iSegReg The register number.
3881	*/
3882	static VBOXSTRICTRC iemMemSegCheckReadAccessEx(PIEMCPU pIemCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg)
3883	{
3884	if (!pHid->Attr.n.u1Present)
3885	return iemRaiseSelectorNotPresentBySegReg(pIemCpu, iSegReg);
3886
3887	if ( (pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE
3888	&& pIemCpu->enmCpuMode != IEMMODE_64BIT )
3889	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, IEM_ACCESS_DATA_R);
3890
3891	/** @todo DPL/RPL/CPL? */
3892
3893	return VINF_SUCCESS;
3894	}
3895
3896
3897	/**
3898	* Applies the segment limit, base and attributes.
3899	*
3900	* This may raise a \#GP or \#SS.
3901	*
3902	* @returns VBox strict status code.
3903	*
3904	* @param pIemCpu The IEM per CPU data.
3905	* @param fAccess The kind of access which is being performed.
3906	* @param iSegReg The index of the segment register to apply.
3907	* This is UINT8_MAX if none (for IDT, GDT, LDT,
3908	* TSS, ++).
3909	* @param pGCPtrMem Pointer to the guest memory address to apply
3910	* segmentation to. Input and output parameter.
3911	*/
3912	static VBOXSTRICTRC iemMemApplySegment(PIEMCPU pIemCpu, uint32_t fAccess, uint8_t iSegReg,
3913	size_t cbMem, PRTGCPTR pGCPtrMem)
3914	{
3915	if (iSegReg == UINT8_MAX)
3916	return VINF_SUCCESS;
3917
3918	PCPUMSELREGHID pSel = iemSRegGetHid(pIemCpu, iSegReg);
3919	switch (pIemCpu->enmCpuMode)
3920	{
3921	case IEMMODE_16BIT:
3922	case IEMMODE_32BIT:
3923	{
3924	RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
3925	RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
3926
3927	Assert(pSel->Attr.n.u1Present);
3928	Assert(pSel->Attr.n.u1DescType);
3929	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
3930	{
3931	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
3932	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3933	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, fAccess);
3934
3935	if (!IEM_IS_REAL_OR_V86_MODE(pIemCpu))
3936	{
3937	/** @todo CPL check. */
3938	}
3939
3940	/*
3941	* There are two kinds of data selectors, normal and expand down.
3942	*/
3943	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
3944	{
3945	if ( GCPtrFirst32 > pSel->u32Limit
3946	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
3947	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
3948
3949	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
3950	}
3951	else
3952	{
3953	/** @todo implement expand down segments. */
3954	AssertFailed(/** @todo implement this */);
3955	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
3956	}
3957	}
3958	else
3959	{
3960
3961	/*
3962	* Code selector and usually be used to read thru, writing is
3963	* only permitted in real and V8086 mode.
3964	*/
3965	if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
3966	\|\| ( (fAccess & IEM_ACCESS_TYPE_READ)
3967	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
3968	&& !IEM_IS_REAL_OR_V86_MODE(pIemCpu) )
3969	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, fAccess);
3970
3971	if ( GCPtrFirst32 > pSel->u32Limit
3972	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
3973	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
3974
3975	if (!IEM_IS_REAL_OR_V86_MODE(pIemCpu))
3976	{
3977	/** @todo CPL check. */
3978	}
3979
3980	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
3981	}
3982	return VINF_SUCCESS;
3983	}
3984
3985	case IEMMODE_64BIT:
3986	if (iSegReg == X86_SREG_GS \|\| iSegReg == X86_SREG_FS)
3987	*pGCPtrMem += pSel->u64Base;
3988	return VINF_SUCCESS;
3989
3990	default:
3991	AssertFailedReturn(VERR_INTERNAL_ERROR_5);
3992	}
3993	}
3994
3995
3996	/**
3997	* Translates a virtual address to a physical physical address and checks if we
3998	* can access the page as specified.
3999	*
4000	* @param pIemCpu The IEM per CPU data.
4001	* @param GCPtrMem The virtual address.
4002	* @param fAccess The intended access.
4003	* @param pGCPhysMem Where to return the physical address.
4004	*/
4005	static VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PIEMCPU pIemCpu, RTGCPTR GCPtrMem, uint32_t fAccess,
4006	PRTGCPHYS pGCPhysMem)
4007	{
4008	/** @todo Need a different PGM interface here. We're currently using
4009	* generic / REM interfaces. this won't cut it for R0 & RC. */
4010	RTGCPHYS GCPhys;
4011	uint64_t fFlags;
4012	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrMem, &fFlags, &GCPhys);
4013	if (RT_FAILURE(rc))
4014	{
4015	/** @todo Check unassigned memory in unpaged mode. */
4016	/** @todo Reserved bits in page tables. Requires new PGM interface. */
4017	*pGCPhysMem = NIL_RTGCPHYS;
4018	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess, rc);
4019	}
4020
4021	/* If the page is writable and does not have the no-exec bit set, all
4022	access is allowed. Otherwise we'll have to check more carefully... */
4023	if ((fFlags & (X86_PTE_RW \| X86_PTE_US \| X86_PTE_PAE_NX)) != (X86_PTE_RW \| X86_PTE_US))
4024	{
4025	/* Write to read only memory? */
4026	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
4027	&& !(fFlags & X86_PTE_RW)
4028	&& ( pIemCpu->uCpl != 0
4029	\|\| (pIemCpu->CTX_SUFF(pCtx)->cr0 & X86_CR0_WP)))
4030	{
4031	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
4032	*pGCPhysMem = NIL_RTGCPHYS;
4033	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
4034	}
4035
4036	/* Kernel memory accessed by userland? */
4037	if ( !(fFlags & X86_PTE_US)
4038	&& pIemCpu->uCpl == 3
4039	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
4040	{
4041	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
4042	*pGCPhysMem = NIL_RTGCPHYS;
4043	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
4044	}
4045
4046	/* Executing non-executable memory? */
4047	if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
4048	&& (fFlags & X86_PTE_PAE_NX)
4049	&& (pIemCpu->CTX_SUFF(pCtx)->msrEFER & MSR_K6_EFER_NXE) )
4050	{
4051	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
4052	*pGCPhysMem = NIL_RTGCPHYS;
4053	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE),
4054	VERR_ACCESS_DENIED);
4055	}
4056	}
4057
4058	GCPhys \|= GCPtrMem & PAGE_OFFSET_MASK;
4059	*pGCPhysMem = GCPhys;
4060	return VINF_SUCCESS;
4061	}
4062
4063
4064
4065	/**
4066	* Maps a physical page.
4067	*
4068	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
4069	* @param pIemCpu The IEM per CPU data.
4070	* @param GCPhysMem The physical address.
4071	* @param fAccess The intended access.
4072	* @param ppvMem Where to return the mapping address.
4073	*/
4074	static int iemMemPageMap(PIEMCPU pIemCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem)
4075	{
4076	#ifdef IEM_VERIFICATION_MODE
4077	/* Force the alternative path so we can ignore writes. */
4078	if ((fAccess & IEM_ACCESS_TYPE_WRITE) && !pIemCpu->fNoRem)
4079	return VERR_PGM_PHYS_TLB_CATCH_ALL;
4080	#endif
4081
4082	/*
4083	* If we can map the page without trouble, do a block processing
4084	* until the end of the current page.
4085	*/
4086	/** @todo need some better API. */
4087	return PGMR3PhysTlbGCPhys2Ptr(IEMCPU_TO_VM(pIemCpu),
4088	GCPhysMem,
4089	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
4090	ppvMem);
4091	}
4092
4093
4094	/**
4095	* Unmap a page previously mapped by iemMemPageMap.
4096	*
4097	* This is currently a dummy function.
4098	*
4099	* @param pIemCpu The IEM per CPU data.
4100	* @param GCPhysMem The physical address.
4101	* @param fAccess The intended access.
4102	* @param pvMem What iemMemPageMap returned.
4103	*/
4104	DECLINLINE(void) iemMemPageUnmap(PIEMCPU pIemCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem)
4105	{
4106	NOREF(pIemCpu);
4107	NOREF(GCPhysMem);
4108	NOREF(fAccess);
4109	NOREF(pvMem);
4110	}
4111
4112
4113	/**
4114	* Looks up a memory mapping entry.
4115	*
4116	* @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
4117	* @param pIemCpu The IEM per CPU data.
4118	* @param pvMem The memory address.
4119	* @param fAccess The access to.
4120	*/
4121	DECLINLINE(int) iemMapLookup(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess)
4122	{
4123	fAccess &= IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK;
4124	if ( pIemCpu->aMemMappings[0].pv == pvMem
4125	&& (pIemCpu->aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4126	return 0;
4127	if ( pIemCpu->aMemMappings[1].pv == pvMem
4128	&& (pIemCpu->aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4129	return 1;
4130	if ( pIemCpu->aMemMappings[2].pv == pvMem
4131	&& (pIemCpu->aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4132	return 2;
4133	return VERR_NOT_FOUND;
4134	}
4135
4136
4137	/**
4138	* Finds a free memmap entry when using iNextMapping doesn't work.
4139	*
4140	* @returns Memory mapping index, 1024 on failure.
4141	* @param pIemCpu The IEM per CPU data.
4142	*/
4143	static unsigned iemMemMapFindFree(PIEMCPU pIemCpu)
4144	{
4145	/*
4146	* The easy case.
4147	*/
4148	if (pIemCpu->cActiveMappings == 0)
4149	{
4150	pIemCpu->iNextMapping = 1;
4151	return 0;
4152	}
4153
4154	/* There should be enough mappings for all instructions. */
4155	AssertReturn(pIemCpu->cActiveMappings < RT_ELEMENTS(pIemCpu->aMemMappings), 1024);
4156
4157	for (unsigned i = 0; i < RT_ELEMENTS(pIemCpu->aMemMappings); i++)
4158	if (pIemCpu->aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
4159	return i;
4160
4161	AssertFailedReturn(1024);
4162	}
4163
4164
4165	/**
4166	* Commits a bounce buffer that needs writing back and unmaps it.
4167	*
4168	* @returns Strict VBox status code.
4169	* @param pIemCpu The IEM per CPU data.
4170	* @param iMemMap The index of the buffer to commit.
4171	*/
4172	static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PIEMCPU pIemCpu, unsigned iMemMap)
4173	{
4174	Assert(pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
4175	Assert(pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
4176
4177	/*
4178	* Do the writing.
4179	*/
4180	int rc;
4181	if ( !pIemCpu->aMemBbMappings[iMemMap].fUnassigned
4182	&& !IEM_VERIFICATION_ENABLED(pIemCpu))
4183	{
4184	uint16_t const cbFirst = pIemCpu->aMemBbMappings[iMemMap].cbFirst;
4185	uint16_t const cbSecond = pIemCpu->aMemBbMappings[iMemMap].cbSecond;
4186	uint8_t const *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4187	if (!pIemCpu->fByPassHandlers)
4188	{
4189	rc = PGMPhysWrite(IEMCPU_TO_VM(pIemCpu),
4190	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
4191	pbBuf,
4192	cbFirst);
4193	if (cbSecond && rc == VINF_SUCCESS)
4194	rc = PGMPhysWrite(IEMCPU_TO_VM(pIemCpu),
4195	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
4196	pbBuf + cbFirst,
4197	cbSecond);
4198	}
4199	else
4200	{
4201	rc = PGMPhysSimpleWriteGCPhys(IEMCPU_TO_VM(pIemCpu),
4202	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
4203	pbBuf,
4204	cbFirst);
4205	if (cbSecond && rc == VINF_SUCCESS)
4206	rc = PGMPhysSimpleWriteGCPhys(IEMCPU_TO_VM(pIemCpu),
4207	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
4208	pbBuf + cbFirst,
4209	cbSecond);
4210	}
4211	}
4212	else
4213	rc = VINF_SUCCESS;
4214
4215	#ifdef IEM_VERIFICATION_MODE
4216	/*
4217	* Record the write(s).
4218	*/
4219	if (!pIemCpu->fNoRem)
4220	{
4221	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4222	if (pEvtRec)
4223	{
4224	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
4225	pEvtRec->u.RamWrite.GCPhys = pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst;
4226	pEvtRec->u.RamWrite.cb = pIemCpu->aMemBbMappings[iMemMap].cbFirst;
4227	memcpy(pEvtRec->u.RamWrite.ab, &pIemCpu->aBounceBuffers[iMemMap].ab[0], pIemCpu->aMemBbMappings[iMemMap].cbFirst);
4228	AssertCompile(sizeof(pEvtRec->u.RamWrite.ab) == sizeof(pIemCpu->aBounceBuffers[0].ab));
4229	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4230	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4231	}
4232	if (pIemCpu->aMemBbMappings[iMemMap].cbSecond)
4233	{
4234	pEvtRec = iemVerifyAllocRecord(pIemCpu);
4235	if (pEvtRec)
4236	{
4237	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
4238	pEvtRec->u.RamWrite.GCPhys = pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond;
4239	pEvtRec->u.RamWrite.cb = pIemCpu->aMemBbMappings[iMemMap].cbSecond;
4240	memcpy(pEvtRec->u.RamWrite.ab,
4241	&pIemCpu->aBounceBuffers[iMemMap].ab[pIemCpu->aMemBbMappings[iMemMap].cbFirst],
4242	pIemCpu->aMemBbMappings[iMemMap].cbSecond);
4243	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4244	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4245	}
4246	}
4247	}
4248	#endif
4249
4250	/*
4251	* Free the mapping entry.
4252	*/
4253	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
4254	Assert(pIemCpu->cActiveMappings != 0);
4255	pIemCpu->cActiveMappings--;
4256	return rc;
4257	}
4258
4259
4260	/**
4261	* iemMemMap worker that deals with a request crossing pages.
4262	*/
4263	static VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PIEMCPU pIemCpu, int iMemMap, void **ppvMem,
4264	size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
4265	{
4266	/*
4267	* Do the address translations.
4268	*/
4269	RTGCPHYS GCPhysFirst;
4270	VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrFirst, fAccess, &GCPhysFirst);
4271	if (rcStrict != VINF_SUCCESS)
4272	return rcStrict;
4273
4274	RTGCPHYS GCPhysSecond;
4275	rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrFirst + (cbMem - 1), fAccess, &GCPhysSecond);
4276	if (rcStrict != VINF_SUCCESS)
4277	return rcStrict;
4278	GCPhysSecond &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
4279
4280	/*
4281	* Read in the current memory content if it's a read, execute or partial
4282	* write access.
4283	*/
4284	uint8_t *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4285	uint32_t const cbFirstPage = PAGE_SIZE - (GCPhysFirst & PAGE_OFFSET_MASK);
4286	uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
4287
4288	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
4289	{
4290	int rc;
4291	if (!pIemCpu->fByPassHandlers)
4292	{
4293	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysFirst, pbBuf, cbFirstPage);
4294	if (rc != VINF_SUCCESS)
4295	return rc;
4296	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage);
4297	if (rc != VINF_SUCCESS)
4298	return rc;
4299	}
4300	else
4301	{
4302	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf, GCPhysFirst, cbFirstPage);
4303	if (rc != VINF_SUCCESS)
4304	return rc;
4305	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
4306	if (rc != VINF_SUCCESS)
4307	return rc;
4308	}
4309
4310	#ifdef IEM_VERIFICATION_MODE
4311	if ( !pIemCpu->fNoRem
4312	&& (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC)) )
4313	{
4314	/*
4315	* Record the reads.
4316	*/
4317	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4318	if (pEvtRec)
4319	{
4320	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4321	pEvtRec->u.RamRead.GCPhys = GCPhysFirst;
4322	pEvtRec->u.RamRead.cb = cbFirstPage;
4323	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4324	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4325	}
4326	pEvtRec = iemVerifyAllocRecord(pIemCpu);
4327	if (pEvtRec)
4328	{
4329	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4330	pEvtRec->u.RamRead.GCPhys = GCPhysSecond;
4331	pEvtRec->u.RamRead.cb = cbSecondPage;
4332	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4333	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4334	}
4335	}
4336	#endif
4337	}
4338	#ifdef VBOX_STRICT
4339	else
4340	memset(pbBuf, 0xcc, cbMem);
4341	#endif
4342	#ifdef VBOX_STRICT
4343	if (cbMem < sizeof(pIemCpu->aBounceBuffers[iMemMap].ab))
4344	memset(pbBuf + cbMem, 0xaa, sizeof(pIemCpu->aBounceBuffers[iMemMap].ab) - cbMem);
4345	#endif
4346
4347	/*
4348	* Commit the bounce buffer entry.
4349	*/
4350	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
4351	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
4352	pIemCpu->aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
4353	pIemCpu->aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
4354	pIemCpu->aMemBbMappings[iMemMap].fUnassigned = false;
4355	pIemCpu->aMemMappings[iMemMap].pv = pbBuf;
4356	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
4357	pIemCpu->cActiveMappings++;
4358
4359	*ppvMem = pbBuf;
4360	return VINF_SUCCESS;
4361	}
4362
4363
4364	/**
4365	* iemMemMap woker that deals with iemMemPageMap failures.
4366	*/
4367	static VBOXSTRICTRC iemMemBounceBufferMapPhys(PIEMCPU pIemCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
4368	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
4369	{
4370	/*
4371	* Filter out conditions we can handle and the ones which shouldn't happen.
4372	*/
4373	if ( rcMap != VINF_PGM_PHYS_TLB_CATCH_WRITE
4374	&& rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
4375	&& rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
4376	{
4377	AssertReturn(RT_FAILURE_NP(rcMap), VERR_INTERNAL_ERROR_3);
4378	return rcMap;
4379	}
4380	pIemCpu->cPotentialExits++;
4381
4382	/*
4383	* Read in the current memory content if it's a read, execute or partial
4384	* write access.
4385	*/
4386	uint8_t *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4387	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
4388	{
4389	if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
4390	memset(pbBuf, 0xff, cbMem);
4391	else
4392	{
4393	int rc;
4394	if (!pIemCpu->fByPassHandlers)
4395	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysFirst, pbBuf, cbMem);
4396	else
4397	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf, GCPhysFirst, cbMem);
4398	if (rc != VINF_SUCCESS)
4399	return rc;
4400	}
4401
4402	#ifdef IEM_VERIFICATION_MODE
4403	if ( !pIemCpu->fNoRem
4404	&& (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC)) )
4405	{
4406	/*
4407	* Record the read.
4408	*/
4409	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4410	if (pEvtRec)
4411	{
4412	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4413	pEvtRec->u.RamRead.GCPhys = GCPhysFirst;
4414	pEvtRec->u.RamRead.cb = (uint32_t)cbMem;
4415	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4416	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4417	}
4418	}
4419	#endif
4420	}
4421	#ifdef VBOX_STRICT
4422	else
4423	memset(pbBuf, 0xcc, cbMem);
4424	#endif
4425	#ifdef VBOX_STRICT
4426	if (cbMem < sizeof(pIemCpu->aBounceBuffers[iMemMap].ab))
4427	memset(pbBuf + cbMem, 0xaa, sizeof(pIemCpu->aBounceBuffers[iMemMap].ab) - cbMem);
4428	#endif
4429
4430	/*
4431	* Commit the bounce buffer entry.
4432	*/
4433	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
4434	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
4435	pIemCpu->aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
4436	pIemCpu->aMemBbMappings[iMemMap].cbSecond = 0;
4437	pIemCpu->aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
4438	pIemCpu->aMemMappings[iMemMap].pv = pbBuf;
4439	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
4440	pIemCpu->cActiveMappings++;
4441
4442	*ppvMem = pbBuf;
4443	return VINF_SUCCESS;
4444	}
4445
4446
4447
4448	/**
4449	* Maps the specified guest memory for the given kind of access.
4450	*
4451	* This may be using bounce buffering of the memory if it's crossing a page
4452	* boundary or if there is an access handler installed for any of it. Because
4453	* of lock prefix guarantees, we're in for some extra clutter when this
4454	* happens.
4455	*
4456	* This may raise a \#GP, \#SS, \#PF or \#AC.
4457	*
4458	* @returns VBox strict status code.
4459	*
4460	* @param pIemCpu The IEM per CPU data.
4461	* @param ppvMem Where to return the pointer to the mapped
4462	* memory.
4463	* @param cbMem The number of bytes to map. This is usually 1,
4464	* 2, 4, 6, 8, 12, 16, 32 or 512. When used by
4465	* string operations it can be up to a page.
4466	* @param iSegReg The index of the segment register to use for
4467	* this access. The base and limits are checked.
4468	* Use UINT8_MAX to indicate that no segmentation
4469	* is required (for IDT, GDT and LDT accesses).
4470	* @param GCPtrMem The address of the guest memory.
4471	* @param a_fAccess How the memory is being accessed. The
4472	* IEM_ACCESS_TYPE_XXX bit is used to figure out
4473	* how to map the memory, while the
4474	* IEM_ACCESS_WHAT_XXX bit is used when raising
4475	* exceptions.
4476	*/
4477	static VBOXSTRICTRC iemMemMap(PIEMCPU pIemCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
4478	{
4479	/*
4480	* Check the input and figure out which mapping entry to use.
4481	*/
4482	Assert(cbMem <= 32 \|\| cbMem == 512);
4483	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
4484
4485	unsigned iMemMap = pIemCpu->iNextMapping;
4486	if (iMemMap >= RT_ELEMENTS(pIemCpu->aMemMappings))
4487	{
4488	iMemMap = iemMemMapFindFree(pIemCpu);
4489	AssertReturn(iMemMap < RT_ELEMENTS(pIemCpu->aMemMappings), VERR_INTERNAL_ERROR_3);
4490	}
4491
4492	/*
4493	* Map the memory, checking that we can actually access it. If something
4494	* slightly complicated happens, fall back on bounce buffering.
4495	*/
4496	VBOXSTRICTRC rcStrict = iemMemApplySegment(pIemCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
4497	if (rcStrict != VINF_SUCCESS)
4498	return rcStrict;
4499
4500	if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem > PAGE_SIZE) /* Crossing a page boundary? */
4501	return iemMemBounceBufferMapCrossPage(pIemCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
4502
4503	RTGCPHYS GCPhysFirst;
4504	rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrMem, fAccess, &GCPhysFirst);
4505	if (rcStrict != VINF_SUCCESS)
4506	return rcStrict;
4507
4508	void *pvMem;
4509	rcStrict = iemMemPageMap(pIemCpu, GCPhysFirst, fAccess, &pvMem);
4510	if (rcStrict != VINF_SUCCESS)
4511	return iemMemBounceBufferMapPhys(pIemCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
4512
4513	/*
4514	* Fill in the mapping table entry.
4515	*/
4516	pIemCpu->aMemMappings[iMemMap].pv = pvMem;
4517	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess;
4518	pIemCpu->iNextMapping = iMemMap + 1;
4519	pIemCpu->cActiveMappings++;
4520
4521	*ppvMem = pvMem;
4522	return VINF_SUCCESS;
4523	}
4524
4525
4526	/**
4527	* Commits the guest memory if bounce buffered and unmaps it.
4528	*
4529	* @returns Strict VBox status code.
4530	* @param pIemCpu The IEM per CPU data.
4531	* @param pvMem The mapping.
4532	* @param fAccess The kind of access.
4533	*/
4534	static VBOXSTRICTRC iemMemCommitAndUnmap(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess)
4535	{
4536	int iMemMap = iemMapLookup(pIemCpu, pvMem, fAccess);
4537	AssertReturn(iMemMap >= 0, iMemMap);
4538
4539	/*
4540	* If it's bounce buffered, we need to write back the buffer.
4541	*/
4542	if ( (pIemCpu->aMemMappings[iMemMap].fAccess & (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_TYPE_WRITE))
4543	== (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_TYPE_WRITE))
4544	return iemMemBounceBufferCommitAndUnmap(pIemCpu, iMemMap);
4545
4546	/* Free the entry. */
4547	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
4548	Assert(pIemCpu->cActiveMappings != 0);
4549	pIemCpu->cActiveMappings--;
4550	return VINF_SUCCESS;
4551	}
4552
4553
4554	/**
4555	* Fetches a data byte.
4556	*
4557	* @returns Strict VBox status code.
4558	* @param pIemCpu The IEM per CPU data.
4559	* @param pu8Dst Where to return the byte.
4560	* @param iSegReg The index of the segment register to use for
4561	* this access. The base and limits are checked.
4562	* @param GCPtrMem The address of the guest memory.
4563	*/
4564	static VBOXSTRICTRC iemMemFetchDataU8(PIEMCPU pIemCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4565	{
4566	/* The lazy approach for now... */
4567	uint8_t const *pu8Src;
4568	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu8Src, sizeof(pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4569	if (rc == VINF_SUCCESS)
4570	{
4571	pu8Dst = pu8Src;
4572	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
4573	}
4574	return rc;
4575	}
4576
4577
4578	/**
4579	* Fetches a data word.
4580	*
4581	* @returns Strict VBox status code.
4582	* @param pIemCpu The IEM per CPU data.
4583	* @param pu16Dst Where to return the word.
4584	* @param iSegReg The index of the segment register to use for
4585	* this access. The base and limits are checked.
4586	* @param GCPtrMem The address of the guest memory.
4587	*/
4588	static VBOXSTRICTRC iemMemFetchDataU16(PIEMCPU pIemCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4589	{
4590	/* The lazy approach for now... */
4591	uint16_t const *pu16Src;
4592	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4593	if (rc == VINF_SUCCESS)
4594	{
4595	pu16Dst = pu16Src;
4596	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
4597	}
4598	return rc;
4599	}
4600
4601
4602	/**
4603	* Fetches a data dword.
4604	*
4605	* @returns Strict VBox status code.
4606	* @param pIemCpu The IEM per CPU data.
4607	* @param pu32Dst Where to return the dword.
4608	* @param iSegReg The index of the segment register to use for
4609	* this access. The base and limits are checked.
4610	* @param GCPtrMem The address of the guest memory.
4611	*/
4612	static VBOXSTRICTRC iemMemFetchDataU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4613	{
4614	/* The lazy approach for now... */
4615	uint32_t const *pu32Src;
4616	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4617	if (rc == VINF_SUCCESS)
4618	{
4619	pu32Dst = pu32Src;
4620	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
4621	}
4622	return rc;
4623	}
4624
4625
4626	#ifdef SOME_UNUSED_FUNCTION
4627	/**
4628	* Fetches a data dword and sign extends it to a qword.
4629	*
4630	* @returns Strict VBox status code.
4631	* @param pIemCpu The IEM per CPU data.
4632	* @param pu64Dst Where to return the sign extended value.
4633	* @param iSegReg The index of the segment register to use for
4634	* this access. The base and limits are checked.
4635	* @param GCPtrMem The address of the guest memory.
4636	*/
4637	static VBOXSTRICTRC iemMemFetchDataS32SxU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4638	{
4639	/* The lazy approach for now... */
4640	int32_t const *pi32Src;
4641	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pi32Src, sizeof(pi32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4642	if (rc == VINF_SUCCESS)
4643	{
4644	pu64Dst = pi32Src;
4645	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
4646	}
4647	#ifdef __GNUC__ /* warning: GCC may be a royal pain */
4648	else
4649	*pu64Dst = 0;
4650	#endif
4651	return rc;
4652	}
4653	#endif
4654
4655
4656	/**
4657	* Fetches a data qword.
4658	*
4659	* @returns Strict VBox status code.
4660	* @param pIemCpu The IEM per CPU data.
4661	* @param pu64Dst Where to return the qword.
4662	* @param iSegReg The index of the segment register to use for
4663	* this access. The base and limits are checked.
4664	* @param GCPtrMem The address of the guest memory.
4665	*/
4666	static VBOXSTRICTRC iemMemFetchDataU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4667	{
4668	/* The lazy approach for now... */
4669	uint64_t const *pu64Src;
4670	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4671	if (rc == VINF_SUCCESS)
4672	{
4673	pu64Dst = pu64Src;
4674	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
4675	}
4676	return rc;
4677	}
4678
4679
4680	/**
4681	* Fetches a descriptor register (lgdt, lidt).
4682	*
4683	* @returns Strict VBox status code.
4684	* @param pIemCpu The IEM per CPU data.
4685	* @param pcbLimit Where to return the limit.
4686	* @param pGCPTrBase Where to return the base.
4687	* @param iSegReg The index of the segment register to use for
4688	* this access. The base and limits are checked.
4689	* @param GCPtrMem The address of the guest memory.
4690	* @param enmOpSize The effective operand size.
4691	*/
4692	static VBOXSTRICTRC iemMemFetchDataXdtr(PIEMCPU pIemCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase,
4693	uint8_t iSegReg, RTGCPTR GCPtrMem, IEMMODE enmOpSize)
4694	{
4695	uint8_t const *pu8Src;
4696	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu,
4697	(void **)&pu8Src,
4698	enmOpSize == IEMMODE_64BIT
4699	? 2 + 8
4700	: enmOpSize == IEMMODE_32BIT
4701	? 2 + 4
4702	: 2 + 3,
4703	iSegReg,
4704	GCPtrMem,
4705	IEM_ACCESS_DATA_R);
4706	if (rcStrict == VINF_SUCCESS)
4707	{
4708	*pcbLimit = RT_MAKE_U16(pu8Src[0], pu8Src[1]);
4709	switch (enmOpSize)
4710	{
4711	case IEMMODE_16BIT:
4712	*pGCPtrBase = RT_MAKE_U32_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], 0);
4713	break;
4714	case IEMMODE_32BIT:
4715	*pGCPtrBase = RT_MAKE_U32_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], pu8Src[5]);
4716	break;
4717	case IEMMODE_64BIT:
4718	*pGCPtrBase = RT_MAKE_U64_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], pu8Src[5],
4719	pu8Src[6], pu8Src[7], pu8Src[8], pu8Src[9]);
4720	break;
4721
4722	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4723	}
4724	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
4725	}
4726	return rcStrict;
4727	}
4728
4729
4730
4731	/**
4732	* Stores a data byte.
4733	*
4734	* @returns Strict VBox status code.
4735	* @param pIemCpu The IEM per CPU data.
4736	* @param iSegReg The index of the segment register to use for
4737	* this access. The base and limits are checked.
4738	* @param GCPtrMem The address of the guest memory.
4739	* @param u8Value The value to store.
4740	*/
4741	static VBOXSTRICTRC iemMemStoreDataU8(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
4742	{
4743	/* The lazy approach for now... */
4744	uint8_t *pu8Dst;
4745	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu8Dst, sizeof(pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
4746	if (rc == VINF_SUCCESS)
4747	{
4748	*pu8Dst = u8Value;
4749	rc = iemMemCommitAndUnmap(pIemCpu, pu8Dst, IEM_ACCESS_DATA_W);
4750	}
4751	return rc;
4752	}
4753
4754
4755	/**
4756	* Stores a data word.
4757	*
4758	* @returns Strict VBox status code.
4759	* @param pIemCpu The IEM per CPU data.
4760	* @param iSegReg The index of the segment register to use for
4761	* this access. The base and limits are checked.
4762	* @param GCPtrMem The address of the guest memory.
4763	* @param u16Value The value to store.
4764	*/
4765	static VBOXSTRICTRC iemMemStoreDataU16(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
4766	{
4767	/* The lazy approach for now... */
4768	uint16_t *pu16Dst;
4769	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
4770	if (rc == VINF_SUCCESS)
4771	{
4772	*pu16Dst = u16Value;
4773	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_DATA_W);
4774	}
4775	return rc;
4776	}
4777
4778
4779	/**
4780	* Stores a data dword.
4781	*
4782	* @returns Strict VBox status code.
4783	* @param pIemCpu The IEM per CPU data.
4784	* @param iSegReg The index of the segment register to use for
4785	* this access. The base and limits are checked.
4786	* @param GCPtrMem The address of the guest memory.
4787	* @param u32Value The value to store.
4788	*/
4789	static VBOXSTRICTRC iemMemStoreDataU32(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
4790	{
4791	/* The lazy approach for now... */
4792	uint32_t *pu32Dst;
4793	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
4794	if (rc == VINF_SUCCESS)
4795	{
4796	*pu32Dst = u32Value;
4797	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_DATA_W);
4798	}
4799	return rc;
4800	}
4801
4802
4803	/**
4804	* Stores a data qword.
4805	*
4806	* @returns Strict VBox status code.
4807	* @param pIemCpu The IEM per CPU data.
4808	* @param iSegReg The index of the segment register to use for
4809	* this access. The base and limits are checked.
4810	* @param GCPtrMem The address of the guest memory.
4811	* @param u64Value The value to store.
4812	*/
4813	static VBOXSTRICTRC iemMemStoreDataU64(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
4814	{
4815	/* The lazy approach for now... */
4816	uint64_t *pu64Dst;
4817	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
4818	if (rc == VINF_SUCCESS)
4819	{
4820	*pu64Dst = u64Value;
4821	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_DATA_W);
4822	}
4823	return rc;
4824	}
4825
4826
4827	/**
4828	* Pushes a word onto the stack.
4829	*
4830	* @returns Strict VBox status code.
4831	* @param pIemCpu The IEM per CPU data.
4832	* @param u16Value The value to push.
4833	*/
4834	static VBOXSTRICTRC iemMemStackPushU16(PIEMCPU pIemCpu, uint16_t u16Value)
4835	{
4836	/* Increment the stack pointer. */
4837	uint64_t uNewRsp;
4838	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4839	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 2, &uNewRsp);
4840
4841	/* Write the word the lazy way. */
4842	uint16_t *pu16Dst;
4843	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
4844	if (rc == VINF_SUCCESS)
4845	{
4846	*pu16Dst = u16Value;
4847	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_W);
4848	}
4849
4850	/* Commit the new RSP value unless we an access handler made trouble. */
4851	if (rc == VINF_SUCCESS)
4852	pCtx->rsp = uNewRsp;
4853
4854	return rc;
4855	}
4856
4857
4858	/**
4859	* Pushes a dword onto the stack.
4860	*
4861	* @returns Strict VBox status code.
4862	* @param pIemCpu The IEM per CPU data.
4863	* @param u32Value The value to push.
4864	*/
4865	static VBOXSTRICTRC iemMemStackPushU32(PIEMCPU pIemCpu, uint32_t u32Value)
4866	{
4867	/* Increment the stack pointer. */
4868	uint64_t uNewRsp;
4869	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4870	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 4, &uNewRsp);
4871
4872	/* Write the word the lazy way. */
4873	uint32_t *pu32Dst;
4874	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
4875	if (rc == VINF_SUCCESS)
4876	{
4877	*pu32Dst = u32Value;
4878	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
4879	}
4880
4881	/* Commit the new RSP value unless we an access handler made trouble. */
4882	if (rc == VINF_SUCCESS)
4883	pCtx->rsp = uNewRsp;
4884
4885	return rc;
4886	}
4887
4888
4889	/**
4890	* Pushes a qword onto the stack.
4891	*
4892	* @returns Strict VBox status code.
4893	* @param pIemCpu The IEM per CPU data.
4894	* @param u64Value The value to push.
4895	*/
4896	static VBOXSTRICTRC iemMemStackPushU64(PIEMCPU pIemCpu, uint64_t u64Value)
4897	{
4898	/* Increment the stack pointer. */
4899	uint64_t uNewRsp;
4900	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4901	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 8, &uNewRsp);
4902
4903	/* Write the word the lazy way. */
4904	uint64_t *pu64Dst;
4905	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
4906	if (rc == VINF_SUCCESS)
4907	{
4908	*pu64Dst = u64Value;
4909	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_STACK_W);
4910	}
4911
4912	/* Commit the new RSP value unless we an access handler made trouble. */
4913	if (rc == VINF_SUCCESS)
4914	pCtx->rsp = uNewRsp;
4915
4916	return rc;
4917	}
4918
4919
4920	/**
4921	* Pops a word from the stack.
4922	*
4923	* @returns Strict VBox status code.
4924	* @param pIemCpu The IEM per CPU data.
4925	* @param pu16Value Where to store the popped value.
4926	*/
4927	static VBOXSTRICTRC iemMemStackPopU16(PIEMCPU pIemCpu, uint16_t *pu16Value)
4928	{
4929	/* Increment the stack pointer. */
4930	uint64_t uNewRsp;
4931	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4932	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 2, &uNewRsp);
4933
4934	/* Write the word the lazy way. */
4935	uint16_t const *pu16Src;
4936	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
4937	if (rc == VINF_SUCCESS)
4938	{
4939	pu16Value = pu16Src;
4940	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
4941
4942	/* Commit the new RSP value. */
4943	if (rc == VINF_SUCCESS)
4944	pCtx->rsp = uNewRsp;
4945	}
4946
4947	return rc;
4948	}
4949
4950
4951	/**
4952	* Pops a dword from the stack.
4953	*
4954	* @returns Strict VBox status code.
4955	* @param pIemCpu The IEM per CPU data.
4956	* @param pu32Value Where to store the popped value.
4957	*/
4958	static VBOXSTRICTRC iemMemStackPopU32(PIEMCPU pIemCpu, uint32_t *pu32Value)
4959	{
4960	/* Increment the stack pointer. */
4961	uint64_t uNewRsp;
4962	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4963	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 4, &uNewRsp);
4964
4965	/* Write the word the lazy way. */
4966	uint32_t const *pu32Src;
4967	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
4968	if (rc == VINF_SUCCESS)
4969	{
4970	pu32Value = pu32Src;
4971	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
4972
4973	/* Commit the new RSP value. */
4974	if (rc == VINF_SUCCESS)
4975	pCtx->rsp = uNewRsp;
4976	}
4977
4978	return rc;
4979	}
4980
4981
4982	/**
4983	* Pops a qword from the stack.
4984	*
4985	* @returns Strict VBox status code.
4986	* @param pIemCpu The IEM per CPU data.
4987	* @param pu64Value Where to store the popped value.
4988	*/
4989	static VBOXSTRICTRC iemMemStackPopU64(PIEMCPU pIemCpu, uint64_t *pu64Value)
4990	{
4991	/* Increment the stack pointer. */
4992	uint64_t uNewRsp;
4993	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4994	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 8, &uNewRsp);
4995
4996	/* Write the word the lazy way. */
4997	uint64_t const *pu64Src;
4998	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
4999	if (rc == VINF_SUCCESS)
5000	{
5001	pu64Value = pu64Src;
5002	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
5003
5004	/* Commit the new RSP value. */
5005	if (rc == VINF_SUCCESS)
5006	pCtx->rsp = uNewRsp;
5007	}
5008
5009	return rc;
5010	}
5011
5012
5013	/**
5014	* Pushes a word onto the stack, using a temporary stack pointer.
5015	*
5016	* @returns Strict VBox status code.
5017	* @param pIemCpu The IEM per CPU data.
5018	* @param u16Value The value to push.
5019	* @param pTmpRsp Pointer to the temporary stack pointer.
5020	*/
5021	static VBOXSTRICTRC iemMemStackPushU16Ex(PIEMCPU pIemCpu, uint16_t u16Value, PRTUINT64U pTmpRsp)
5022	{
5023	/* Increment the stack pointer. */
5024	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5025	RTUINT64U NewRsp = *pTmpRsp;
5026	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 2, pCtx);
5027
5028	/* Write the word the lazy way. */
5029	uint16_t *pu16Dst;
5030	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5031	if (rc == VINF_SUCCESS)
5032	{
5033	*pu16Dst = u16Value;
5034	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_W);
5035	}
5036
5037	/* Commit the new RSP value unless we an access handler made trouble. */
5038	if (rc == VINF_SUCCESS)
5039	*pTmpRsp = NewRsp;
5040
5041	return rc;
5042	}
5043
5044
5045	/**
5046	* Pushes a dword onto the stack, using a temporary stack pointer.
5047	*
5048	* @returns Strict VBox status code.
5049	* @param pIemCpu The IEM per CPU data.
5050	* @param u32Value The value to push.
5051	* @param pTmpRsp Pointer to the temporary stack pointer.
5052	*/
5053	static VBOXSTRICTRC iemMemStackPushU32Ex(PIEMCPU pIemCpu, uint32_t u32Value, PRTUINT64U pTmpRsp)
5054	{
5055	/* Increment the stack pointer. */
5056	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5057	RTUINT64U NewRsp = *pTmpRsp;
5058	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 4, pCtx);
5059
5060	/* Write the word the lazy way. */
5061	uint32_t *pu32Dst;
5062	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5063	if (rc == VINF_SUCCESS)
5064	{
5065	*pu32Dst = u32Value;
5066	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
5067	}
5068
5069	/* Commit the new RSP value unless we an access handler made trouble. */
5070	if (rc == VINF_SUCCESS)
5071	*pTmpRsp = NewRsp;
5072
5073	return rc;
5074	}
5075
5076
5077	#ifdef SOME_UNUSED_FUNCTION
5078	/**
5079	* Pushes a dword onto the stack, using a temporary stack pointer.
5080	*
5081	* @returns Strict VBox status code.
5082	* @param pIemCpu The IEM per CPU data.
5083	* @param u64Value The value to push.
5084	* @param pTmpRsp Pointer to the temporary stack pointer.
5085	*/
5086	static VBOXSTRICTRC iemMemStackPushU64Ex(PIEMCPU pIemCpu, uint64_t u64Value, PRTUINT64U pTmpRsp)
5087	{
5088	/* Increment the stack pointer. */
5089	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5090	RTUINT64U NewRsp = *pTmpRsp;
5091	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 8, pCtx);
5092
5093	/* Write the word the lazy way. */
5094	uint64_t *pu64Dst;
5095	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5096	if (rc == VINF_SUCCESS)
5097	{
5098	*pu64Dst = u64Value;
5099	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_STACK_W);
5100	}
5101
5102	/* Commit the new RSP value unless we an access handler made trouble. */
5103	if (rc == VINF_SUCCESS)
5104	*pTmpRsp = NewRsp;
5105
5106	return rc;
5107	}
5108	#endif
5109
5110
5111	/**
5112	* Pops a word from the stack, using a temporary stack pointer.
5113	*
5114	* @returns Strict VBox status code.
5115	* @param pIemCpu The IEM per CPU data.
5116	* @param pu16Value Where to store the popped value.
5117	* @param pTmpRsp Pointer to the temporary stack pointer.
5118	*/
5119	static VBOXSTRICTRC iemMemStackPopU16Ex(PIEMCPU pIemCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp)
5120	{
5121	/* Increment the stack pointer. */
5122	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5123	RTUINT64U NewRsp = *pTmpRsp;
5124	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 2, pCtx);
5125
5126	/* Write the word the lazy way. */
5127	uint16_t const *pu16Src;
5128	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5129	if (rc == VINF_SUCCESS)
5130	{
5131	pu16Value = pu16Src;
5132	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
5133
5134	/* Commit the new RSP value. */
5135	if (rc == VINF_SUCCESS)
5136	*pTmpRsp = NewRsp;
5137	}
5138
5139	return rc;
5140	}
5141
5142
5143	/**
5144	* Pops a dword from the stack, using a temporary stack pointer.
5145	*
5146	* @returns Strict VBox status code.
5147	* @param pIemCpu The IEM per CPU data.
5148	* @param pu32Value Where to store the popped value.
5149	* @param pTmpRsp Pointer to the temporary stack pointer.
5150	*/
5151	static VBOXSTRICTRC iemMemStackPopU32Ex(PIEMCPU pIemCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp)
5152	{
5153	/* Increment the stack pointer. */
5154	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5155	RTUINT64U NewRsp = *pTmpRsp;
5156	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 4, pCtx);
5157
5158	/* Write the word the lazy way. */
5159	uint32_t const *pu32Src;
5160	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5161	if (rc == VINF_SUCCESS)
5162	{
5163	pu32Value = pu32Src;
5164	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
5165
5166	/* Commit the new RSP value. */
5167	if (rc == VINF_SUCCESS)
5168	*pTmpRsp = NewRsp;
5169	}
5170
5171	return rc;
5172	}
5173
5174
5175	/**
5176	* Pops a qword from the stack, using a temporary stack pointer.
5177	*
5178	* @returns Strict VBox status code.
5179	* @param pIemCpu The IEM per CPU data.
5180	* @param pu64Value Where to store the popped value.
5181	* @param pTmpRsp Pointer to the temporary stack pointer.
5182	*/
5183	static VBOXSTRICTRC iemMemStackPopU64Ex(PIEMCPU pIemCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp)
5184	{
5185	/* Increment the stack pointer. */
5186	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5187	RTUINT64U NewRsp = *pTmpRsp;
5188	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 8, pCtx);
5189
5190	/* Write the word the lazy way. */
5191	uint64_t const *pu64Src;
5192	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5193	if (rcStrict == VINF_SUCCESS)
5194	{
5195	pu64Value = pu64Src;
5196	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
5197
5198	/* Commit the new RSP value. */
5199	if (rcStrict == VINF_SUCCESS)
5200	*pTmpRsp = NewRsp;
5201	}
5202
5203	return rcStrict;
5204	}
5205
5206
5207	/**
5208	* Begin a special stack push (used by interrupt, exceptions and such).
5209	*
5210	* This will raise #SS or #PF if appropriate.
5211	*
5212	* @returns Strict VBox status code.
5213	* @param pIemCpu The IEM per CPU data.
5214	* @param cbMem The number of bytes to push onto the stack.
5215	* @param ppvMem Where to return the pointer to the stack memory.
5216	* As with the other memory functions this could be
5217	* direct access or bounce buffered access, so
5218	* don't commit register until the commit call
5219	* succeeds.
5220	* @param puNewRsp Where to return the new RSP value. This must be
5221	* passed unchanged to
5222	* iemMemStackPushCommitSpecial().
5223	*/
5224	static VBOXSTRICTRC iemMemStackPushBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp)
5225	{
5226	Assert(cbMem < UINT8_MAX);
5227	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5228	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, (uint8_t)cbMem, puNewRsp);
5229	return iemMemMap(pIemCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5230	}
5231
5232
5233	/**
5234	* Commits a special stack push (started by iemMemStackPushBeginSpecial).
5235	*
5236	* This will update the rSP.
5237	*
5238	* @returns Strict VBox status code.
5239	* @param pIemCpu The IEM per CPU data.
5240	* @param pvMem The pointer returned by
5241	* iemMemStackPushBeginSpecial().
5242	* @param uNewRsp The new RSP value returned by
5243	* iemMemStackPushBeginSpecial().
5244	*/
5245	static VBOXSTRICTRC iemMemStackPushCommitSpecial(PIEMCPU pIemCpu, void *pvMem, uint64_t uNewRsp)
5246	{
5247	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pIemCpu, pvMem, IEM_ACCESS_STACK_W);
5248	if (rcStrict == VINF_SUCCESS)
5249	pIemCpu->CTX_SUFF(pCtx)->rsp = uNewRsp;
5250	return rcStrict;
5251	}
5252
5253
5254	/**
5255	* Begin a special stack pop (used by iret, retf and such).
5256	*
5257	* This will raise \#SS or \#PF if appropriate.
5258	*
5259	* @returns Strict VBox status code.
5260	* @param pIemCpu The IEM per CPU data.
5261	* @param cbMem The number of bytes to push onto the stack.
5262	* @param ppvMem Where to return the pointer to the stack memory.
5263	* @param puNewRsp Where to return the new RSP value. This must be
5264	* passed unchanged to
5265	* iemMemStackPopCommitSpecial() or applied
5266	* manually if iemMemStackPopDoneSpecial() is used.
5267	*/
5268	static VBOXSTRICTRC iemMemStackPopBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp)
5269	{
5270	Assert(cbMem < UINT8_MAX);
5271	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5272	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, (uint8_t)cbMem, puNewRsp);
5273	return iemMemMap(pIemCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5274	}
5275
5276
5277	/**
5278	* Continue a special stack pop (used by iret and retf).
5279	*
5280	* This will raise \#SS or \#PF if appropriate.
5281	*
5282	* @returns Strict VBox status code.
5283	* @param pIemCpu The IEM per CPU data.
5284	* @param cbMem The number of bytes to push onto the stack.
5285	* @param ppvMem Where to return the pointer to the stack memory.
5286	* @param puNewRsp Where to return the new RSP value. This must be
5287	* passed unchanged to
5288	* iemMemStackPopCommitSpecial() or applied
5289	* manually if iemMemStackPopDoneSpecial() is used.
5290	*/
5291	static VBOXSTRICTRC iemMemStackPopContinueSpecial(PIEMCPU pIemCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp)
5292	{
5293	Assert(cbMem < UINT8_MAX);
5294	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5295	RTUINT64U NewRsp;
5296	NewRsp.u = *puNewRsp;
5297	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 8, pCtx);
5298	*puNewRsp = NewRsp.u;
5299	return iemMemMap(pIemCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5300	}
5301
5302
5303	/**
5304	* Commits a special stack pop (started by iemMemStackPopBeginSpecial).
5305	*
5306	* This will update the rSP.
5307	*
5308	* @returns Strict VBox status code.
5309	* @param pIemCpu The IEM per CPU data.
5310	* @param pvMem The pointer returned by
5311	* iemMemStackPopBeginSpecial().
5312	* @param uNewRsp The new RSP value returned by
5313	* iemMemStackPopBeginSpecial().
5314	*/
5315	static VBOXSTRICTRC iemMemStackPopCommitSpecial(PIEMCPU pIemCpu, void const *pvMem, uint64_t uNewRsp)
5316	{
5317	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
5318	if (rcStrict == VINF_SUCCESS)
5319	pIemCpu->CTX_SUFF(pCtx)->rsp = uNewRsp;
5320	return rcStrict;
5321	}
5322
5323
5324	/**
5325	* Done with a special stack pop (started by iemMemStackPopBeginSpecial or
5326	* iemMemStackPopContinueSpecial).
5327	*
5328	* The caller will manually commit the rSP.
5329	*
5330	* @returns Strict VBox status code.
5331	* @param pIemCpu The IEM per CPU data.
5332	* @param pvMem The pointer returned by
5333	* iemMemStackPopBeginSpecial() or
5334	* iemMemStackPopContinueSpecial().
5335	*/
5336	static VBOXSTRICTRC iemMemStackPopDoneSpecial(PIEMCPU pIemCpu, void const *pvMem)
5337	{
5338	return iemMemCommitAndUnmap(pIemCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
5339	}
5340
5341
5342	/**
5343	* Fetches a system table dword.
5344	*
5345	* @returns Strict VBox status code.
5346	* @param pIemCpu The IEM per CPU data.
5347	* @param pu32Dst Where to return the dword.
5348	* @param iSegReg The index of the segment register to use for
5349	* this access. The base and limits are checked.
5350	* @param GCPtrMem The address of the guest memory.
5351	*/
5352	static VBOXSTRICTRC iemMemFetchSysU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
5353	{
5354	/* The lazy approach for now... */
5355	uint32_t const *pu32Src;
5356	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
5357	if (rc == VINF_SUCCESS)
5358	{
5359	pu32Dst = pu32Src;
5360	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
5361	}
5362	return rc;
5363	}
5364
5365
5366	/**
5367	* Fetches a system table qword.
5368	*
5369	* @returns Strict VBox status code.
5370	* @param pIemCpu The IEM per CPU data.
5371	* @param pu64Dst Where to return the qword.
5372	* @param iSegReg The index of the segment register to use for
5373	* this access. The base and limits are checked.
5374	* @param GCPtrMem The address of the guest memory.
5375	*/
5376	static VBOXSTRICTRC iemMemFetchSysU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
5377	{
5378	/* The lazy approach for now... */
5379	uint64_t const *pu64Src;
5380	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
5381	if (rc == VINF_SUCCESS)
5382	{
5383	pu64Dst = pu64Src;
5384	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
5385	}
5386	return rc;
5387	}
5388
5389
5390	/**
5391	* Fetches a descriptor table entry.
5392	*
5393	* @returns Strict VBox status code.
5394	* @param pIemCpu The IEM per CPU.
5395	* @param pDesc Where to return the descriptor table entry.
5396	* @param uSel The selector which table entry to fetch.
5397	*/
5398	static VBOXSTRICTRC iemMemFetchSelDesc(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel)
5399	{
5400	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5401
5402	/** @todo did the 286 require all 8 bytes to be accessible? */
5403	/*
5404	* Get the selector table base and check bounds.
5405	*/
5406	RTGCPTR GCPtrBase;
5407	if (uSel & X86_SEL_LDT)
5408	{
5409	if ( !pCtx->ldtrHid.Attr.n.u1Present
5410	\|\| (uSel \| 0x7U) > pCtx->ldtrHid.u32Limit )
5411	{
5412	Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
5413	uSel, pCtx->ldtrHid.u32Limit, pCtx->ldtr));
5414	/** @todo is this the right exception? */
5415	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5416	}
5417
5418	Assert(pCtx->ldtrHid.Attr.n.u1Present);
5419	GCPtrBase = pCtx->ldtrHid.u64Base;
5420	}
5421	else
5422	{
5423	if ((uSel \| 0x7U) > pCtx->gdtr.cbGdt)
5424	{
5425	Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pCtx->gdtr.cbGdt));
5426	/** @todo is this the right exception? */
5427	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5428	}
5429	GCPtrBase = pCtx->gdtr.pGdt;
5430	}
5431
5432	/*
5433	* Read the legacy descriptor and maybe the long mode extensions if
5434	* required.
5435	*/
5436	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
5437	if (rcStrict == VINF_SUCCESS)
5438	{
5439	if ( !IEM_IS_LONG_MODE(pIemCpu)
5440	\|\| pDesc->Legacy.Gen.u1DescType)
5441	pDesc->Long.au64[1] = 0;
5442	else if ((uint32_t)(uSel & X86_SEL_MASK) + 15 < (uSel & X86_SEL_LDT ? pCtx->ldtrHid.u32Limit : pCtx->gdtr.cbGdt))
5443	rcStrict = iemMemFetchSysU64(pIemCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
5444	else
5445	{
5446	Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
5447	/** @todo is this the right exception? */
5448	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5449	}
5450	}
5451	return rcStrict;
5452	}
5453
5454
5455	/**
5456	* Fakes a long mode stack selector for SS = 0.
5457	*
5458	* @param pDescSs Where to return the fake stack descriptor.
5459	* @param uDpl The DPL we want.
5460	*/
5461	static void iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
5462	{
5463	pDescSs->Long.au64[0] = 0;
5464	pDescSs->Long.au64[1] = 0;
5465	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
5466	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
5467	pDescSs->Long.Gen.u2Dpl = uDpl;
5468	pDescSs->Long.Gen.u1Present = 1;
5469	pDescSs->Long.Gen.u1Long = 1;
5470	}
5471
5472
5473	/**
5474	* Marks the selector descriptor as accessed (only non-system descriptors).
5475	*
5476	* This function ASSUMES that iemMemFetchSelDesc has be called previously and
5477	* will therefore skip the limit checks.
5478	*
5479	* @returns Strict VBox status code.
5480	* @param pIemCpu The IEM per CPU.
5481	* @param uSel The selector.
5482	*/
5483	static VBOXSTRICTRC iemMemMarkSelDescAccessed(PIEMCPU pIemCpu, uint16_t uSel)
5484	{
5485	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5486
5487	/*
5488	* Get the selector table base and calculate the entry address.
5489	*/
5490	RTGCPTR GCPtr = uSel & X86_SEL_LDT
5491	? pCtx->ldtrHid.u64Base
5492	: pCtx->gdtr.pGdt;
5493	GCPtr += uSel & X86_SEL_MASK;
5494
5495	/*
5496	* ASMAtomicBitSet will assert if the address is misaligned, so do some
5497	* ugly stuff to avoid this. This will make sure it's an atomic access
5498	* as well more or less remove any question about 8-bit or 32-bit accesss.
5499	*/
5500	VBOXSTRICTRC rcStrict;
5501	uint32_t volatile *pu32;
5502	if ((GCPtr & 3) == 0)
5503	{
5504	/* The normal case, map the 32-bit bits around the accessed bit (40). */
5505	GCPtr += 2 + 2;
5506	rcStrict = iemMemMap(pIemCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
5507	if (rcStrict != VINF_SUCCESS)
5508	return rcStrict;
5509	ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
5510	}
5511	else
5512	{
5513	/* The misaligned GDT/LDT case, map the whole thing. */
5514	rcStrict = iemMemMap(pIemCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
5515	if (rcStrict != VINF_SUCCESS)
5516	return rcStrict;
5517	switch ((uintptr_t)pu32 & 3)
5518	{
5519	case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
5520	case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
5521	case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
5522	case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
5523	}
5524	}
5525
5526	return iemMemCommitAndUnmap(pIemCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
5527	}
5528
5529	/** @} */
5530
5531
5532	/*
5533	* Include the C/C++ implementation of instruction.
5534	*/
5535	#include "IEMAllCImpl.cpp.h"
5536
5537
5538
5539	/** @name "Microcode" macros.
5540	*
5541	* The idea is that we should be able to use the same code to interpret
5542	* instructions as well as recompiler instructions. Thus this obfuscation.
5543	*
5544	* @{
5545	*/
5546	#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
5547	#define IEM_MC_END() }
5548	#define IEM_MC_PAUSE() do {} while (0)
5549	#define IEM_MC_CONTINUE() do {} while (0)
5550
5551	/** Internal macro. */
5552	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
5553	do \
5554	{ \
5555	VBOXSTRICTRC rcStrict2 = a_Expr; \
5556	if (rcStrict2 != VINF_SUCCESS) \
5557	return rcStrict2; \
5558	} while (0)
5559
5560	#define IEM_MC_ADVANCE_RIP() iemRegUpdateRip(pIemCpu)
5561	#define IEM_MC_REL_JMP_S8(a_i8) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pIemCpu, a_i8))
5562	#define IEM_MC_REL_JMP_S16(a_i16) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pIemCpu, a_i16))
5563	#define IEM_MC_REL_JMP_S32(a_i32) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pIemCpu, a_i32))
5564	#define IEM_MC_SET_RIP_U16(a_u16NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u16NewIP)))
5565	#define IEM_MC_SET_RIP_U32(a_u32NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u32NewIP)))
5566	#define IEM_MC_SET_RIP_U64(a_u64NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u64NewIP)))
5567
5568	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pIemCpu)
5569	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
5570	do { \
5571	if ((pIemCpu)->CTX_SUFF(pCtx)->cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
5572	return iemRaiseDeviceNotAvailable(pIemCpu); \
5573	} while (0)
5574	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
5575	do { \
5576	if (iemFRegFetchFsw(pIemCpu) & X86_FSW_ES) \
5577	return iemRaiseMathFault(pIemCpu); \
5578	} while (0)
5579	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
5580	do { \
5581	if (pIemCpu->uCpl != 0) \
5582	return iemRaiseGeneralProtectionFault0(pIemCpu); \
5583	} while (0)
5584
5585
5586	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
5587	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
5588	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
5589	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
5590	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
5591	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
5592	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
5593	uint32_t a_Name; \
5594	uint32_t *a_pName = &a_Name
5595	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
5596	do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u = (a_EFlags); Assert((pIemCpu)->CTX_SUFF(pCtx)->eflags.u & X86_EFL_1); } while (0)
5597
5598	#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
5599	#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
5600
5601	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5602	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5603	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5604	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5605	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5606	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5607	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5608	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5609	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5610	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5611	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pIemCpu, (a_iGReg))
5612	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pIemCpu, (a_iGReg))
5613	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pIemCpu, (a_iGReg))
5614	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pIemCpu, (a_iGReg))
5615	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pIemCpu, (a_iGReg))
5616	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pIemCpu, (a_iGReg))
5617	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
5618	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) (a_u16Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5619	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) (a_u32Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5620	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) (a_u64Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5621	#define IEM_MC_FETCH_CR0_U16(a_u16Dst) (a_u16Dst) = (uint16_t)(pIemCpu)->CTX_SUFF(pCtx)->cr0
5622	#define IEM_MC_FETCH_CR0_U32(a_u32Dst) (a_u32Dst) = (uint32_t)(pIemCpu)->CTX_SUFF(pCtx)->cr0
5623	#define IEM_MC_FETCH_CR0_U64(a_u64Dst) (a_u64Dst) = (pIemCpu)->CTX_SUFF(pCtx)->cr0
5624	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = (pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5625	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)(pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5626	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = iemFRegFetchFsw(pIemCpu)
5627
5628	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8(pIemCpu, (a_iGReg)) = (a_u8Value)
5629	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) = (a_u16Value)
5630	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
5631	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) = (a_u64Value)
5632	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
5633	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
5634	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
5635	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
5636	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) &= UINT32_MAX
5637	#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
5638
5639	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8(pIemCpu, (a_iGReg))
5640	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t *)iemGRegRef(pIemCpu, (a_iGReg))
5641	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
5642	* Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
5643	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t *)iemGRegRef(pIemCpu, (a_iGReg))
5644	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t *)iemGRegRef(pIemCpu, (a_iGReg))
5645	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &(pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5646
5647	#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u8Value)
5648	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u16Value)
5649	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
5650	do { \
5651	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5652	*pu32Reg += (a_u32Value); \
5653	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5654	} while (0)
5655	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u64Value)
5656
5657	#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u8Value)
5658	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u16Value)
5659	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
5660	do { \
5661	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5662	*pu32Reg -= (a_u32Value); \
5663	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5664	} while (0)
5665	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u64Value)
5666
5667	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pIemCpu, (a_iGReg)); } while (0)
5668	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pIemCpu, (a_iGReg)); } while (0)
5669	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pIemCpu, (a_iGReg)); } while (0)
5670	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pIemCpu, (a_iGReg)); } while (0)
5671	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
5672	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
5673	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
5674
5675	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
5676	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
5677	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
5678	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
5679
5680	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
5681	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
5682	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
5683
5684	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
5685	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
5686
5687	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
5688	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
5689	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
5690
5691	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
5692	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
5693	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
5694
5695	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
5696
5697	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
5698
5699	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u8Value)
5700	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u16Value)
5701	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
5702	do { \
5703	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5704	*pu32Reg &= (a_u32Value); \
5705	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5706	} while (0)
5707	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u64Value)
5708
5709	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u8Value)
5710	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u16Value)
5711	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
5712	do { \
5713	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5714	*pu32Reg \|= (a_u32Value); \
5715	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5716	} while (0)
5717	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u64Value)
5718
5719
5720	#define IEM_MC_SET_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u \|= (a_fBit); } while (0)
5721	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u &= ~(a_fBit); } while (0)
5722	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u ^= (a_fBit); } while (0)
5723
5724	#define IEM_MC_CLEAR_FSW_EX() do { (pIemCpu)->CTX_SUFF(pCtx)->fpu.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
5725
5726
5727	#define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
5728	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
5729	#define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
5730	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
5731	#define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
5732	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
5733
5734	#define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
5735	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
5736	#define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
5737	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
5738
5739	#define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
5740	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
5741	#define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
5742	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
5743
5744	#define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
5745	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
5746
5747	#define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
5748	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
5749	#define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
5750	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
5751
5752	#define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
5753	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
5754	#define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
5755	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
5756	#define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
5757	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pIemCpu, &(a_r64Dst), (a_iSeg), (a_GCPtrMem)))
5758
5759
5760	#define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
5761	do { \
5762	uint8_t u8Tmp; \
5763	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
5764	(a_u16Dst) = u8Tmp; \
5765	} while (0)
5766	#define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
5767	do { \
5768	uint8_t u8Tmp; \
5769	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
5770	(a_u32Dst) = u8Tmp; \
5771	} while (0)
5772	#define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
5773	do { \
5774	uint8_t u8Tmp; \
5775	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
5776	(a_u64Dst) = u8Tmp; \
5777	} while (0)
5778	#define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
5779	do { \
5780	uint16_t u16Tmp; \
5781	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
5782	(a_u32Dst) = u16Tmp; \
5783	} while (0)
5784	#define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
5785	do { \
5786	uint16_t u16Tmp; \
5787	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
5788	(a_u64Dst) = u16Tmp; \
5789	} while (0)
5790	#define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
5791	do { \
5792	uint32_t u32Tmp; \
5793	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
5794	(a_u64Dst) = u32Tmp; \
5795	} while (0)
5796
5797	#define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
5798	do { \
5799	uint8_t u8Tmp; \
5800	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
5801	(a_u16Dst) = (int8_t)u8Tmp; \
5802	} while (0)
5803	#define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
5804	do { \
5805	uint8_t u8Tmp; \
5806	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
5807	(a_u32Dst) = (int8_t)u8Tmp; \
5808	} while (0)
5809	#define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
5810	do { \
5811	uint8_t u8Tmp; \
5812	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
5813	(a_u64Dst) = (int8_t)u8Tmp; \
5814	} while (0)
5815	#define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
5816	do { \
5817	uint16_t u16Tmp; \
5818	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
5819	(a_u32Dst) = (int16_t)u16Tmp; \
5820	} while (0)
5821	#define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
5822	do { \
5823	uint16_t u16Tmp; \
5824	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
5825	(a_u64Dst) = (int16_t)u16Tmp; \
5826	} while (0)
5827	#define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
5828	do { \
5829	uint32_t u32Tmp; \
5830	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
5831	(a_u64Dst) = (int32_t)u32Tmp; \
5832	} while (0)
5833
5834	#define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
5835	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
5836	#define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
5837	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
5838	#define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
5839	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
5840	#define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
5841	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
5842
5843	#define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
5844	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
5845
5846	#define IEM_MC_PUSH_U16(a_u16Value) \
5847	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pIemCpu, (a_u16Value)))
5848	#define IEM_MC_PUSH_U32(a_u32Value) \
5849	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pIemCpu, (a_u32Value)))
5850	#define IEM_MC_PUSH_U64(a_u64Value) \
5851	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pIemCpu, (a_u64Value)))
5852
5853	#define IEM_MC_POP_U16(a_pu16Value) \
5854	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pIemCpu, (a_pu16Value)))
5855	#define IEM_MC_POP_U32(a_pu32Value) \
5856	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pIemCpu, (a_pu32Value)))
5857	#define IEM_MC_POP_U64(a_pu64Value) \
5858	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pIemCpu, (a_pu64Value)))
5859
5860	/** Maps guest memory for direct or bounce buffered access.
5861	* The purpose is to pass it to an operand implementation, thus the a_iArg.
5862	* @remarks May return.
5863	*/
5864	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
5865	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pIemCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
5866
5867	/** Maps guest memory for direct or bounce buffered access.
5868	* The purpose is to pass it to an operand implementation, thus the a_iArg.
5869	* @remarks May return.
5870	*/
5871	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg) \
5872	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pIemCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
5873
5874	/** Commits the memory and unmaps the guest memory.
5875	* @remarks May return.
5876	*/
5877	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
5878	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pIemCpu, (a_pvMem), (a_fAccess)))
5879
5880	/** Commits the memory and unmaps the guest memory unless the FPU status word
5881	* indicates an exception (FSW.ES).
5882	* @remarks May return (for now anyway).
5883	*/
5884	#define IEM_MC_MEM_COMMIT_AND_UNMAP_UNLESS_FPU_XCPT(a_pvMem, a_fAccess, a_u16FSW) \
5885	do { \
5886	if (!(a_u16FSW & X86_FSW_ES)) \
5887	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pIemCpu, (a_pvMem), (a_fAccess))); \
5888	} while (0)
5889
5890	/** Calculate efficient address from R/M. */
5891	#define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm) \
5892	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pIemCpu, (bRm), &(a_GCPtrEff)))
5893
5894	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
5895	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
5896	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
5897	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
5898	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
5899
5900	/**
5901	* Defers the rest of the instruction emulation to a C implementation routine
5902	* and returns, only taking the standard parameters.
5903	*
5904	* @param a_pfnCImpl The pointer to the C routine.
5905	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
5906	*/
5907	#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode)
5908
5909	/**
5910	* Defers the rest of instruction emulation to a C implementation routine and
5911	* returns, taking one argument in addition to the standard ones.
5912	*
5913	* @param a_pfnCImpl The pointer to the C routine.
5914	* @param a0 The argument.
5915	*/
5916	#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0)
5917
5918	/**
5919	* Defers the rest of the instruction emulation to a C implementation routine
5920	* and returns, taking two arguments in addition to the standard ones.
5921	*
5922	* @param a_pfnCImpl The pointer to the C routine.
5923	* @param a0 The first extra argument.
5924	* @param a1 The second extra argument.
5925	*/
5926	#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1)
5927
5928	/**
5929	* Defers the rest of the instruction emulation to a C implementation routine
5930	* and returns, taking two arguments in addition to the standard ones.
5931	*
5932	* @param a_pfnCImpl The pointer to the C routine.
5933	* @param a0 The first extra argument.
5934	* @param a1 The second extra argument.
5935	* @param a2 The third extra argument.
5936	*/
5937	#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2)
5938
5939	/**
5940	* Defers the rest of the instruction emulation to a C implementation routine
5941	* and returns, taking two arguments in addition to the standard ones.
5942	*
5943	* @param a_pfnCImpl The pointer to the C routine.
5944	* @param a0 The first extra argument.
5945	* @param a1 The second extra argument.
5946	* @param a2 The third extra argument.
5947	* @param a3 The fourth extra argument.
5948	* @param a4 The fifth extra argument.
5949	*/
5950	#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2, a3, a4)
5951
5952	/**
5953	* Defers the entire instruction emulation to a C implementation routine and
5954	* returns, only taking the standard parameters.
5955	*
5956	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
5957	*
5958	* @param a_pfnCImpl The pointer to the C routine.
5959	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
5960	*/
5961	#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode)
5962
5963	/**
5964	* Defers the entire instruction emulation to a C implementation routine and
5965	* returns, taking one argument in addition to the standard ones.
5966	*
5967	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
5968	*
5969	* @param a_pfnCImpl The pointer to the C routine.
5970	* @param a0 The argument.
5971	*/
5972	#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0)
5973
5974	/**
5975	* Defers the entire instruction emulation to a C implementation routine and
5976	* returns, taking two arguments in addition to the standard ones.
5977	*
5978	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
5979	*
5980	* @param a_pfnCImpl The pointer to the C routine.
5981	* @param a0 The first extra argument.
5982	* @param a1 The second extra argument.
5983	*/
5984	#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1)
5985
5986	/**
5987	* Defers the entire instruction emulation to a C implementation routine and
5988	* returns, taking three arguments in addition to the standard ones.
5989	*
5990	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
5991	*
5992	* @param a_pfnCImpl The pointer to the C routine.
5993	* @param a0 The first extra argument.
5994	* @param a1 The second extra argument.
5995	* @param a2 The third extra argument.
5996	*/
5997	#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2)
5998
5999	/**
6000	* Calls a FPU assembly implementation taking two visible arguments.
6001	*
6002	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6003	*
6004	* @param a_pfnAImpl Pointer to the assembly FPU routine.
6005	* @param a0 The first extra argument.
6006	* @param a1 The second extra argument.
6007	*/
6008	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
6009	do { \
6010	iemFpuPrepareUsage(pIemCpu); \
6011	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1)); \
6012	} while (0)
6013
6014	/**
6015	* Calls a FPU assembly implementation taking three visible arguments.
6016	*
6017	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6018	*
6019	* @param a_pfnAImpl Pointer to the assembly FPU routine.
6020	* @param a0 The first extra argument.
6021	* @param a1 The second extra argument.
6022	* @param a2 The third extra argument.
6023	*/
6024	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
6025	do { \
6026	iemFpuPrepareUsage(pIemCpu); \
6027	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1), (a2)); \
6028	} while (0)
6029
6030	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
6031	do { \
6032	(a_FpuData).FSW = (a_FSW); \
6033	(a_FpuData).r80Result = *(a_pr80Value); \
6034	} while (0)
6035
6036	/** Pushes FPU result onto the stack. */
6037	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
6038	iemFpuPushResult(pIemCpu, &a_FpuData)
6039	/** Pushes FPU result onto the stack and sets the FPUDP. */
6040	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
6041	iemFpuPushResultWithMemOp(pIemCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
6042
6043	/** Stores FPU result in a stack register. */
6044	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
6045	iemFpuStoreResult(pIemCpu, &a_FpuData, a_iStReg)
6046	/** Stores FPU result in a stack register and pops the stack. */
6047	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
6048	iemFpuStoreResultThenPop(pIemCpu, &a_FpuData, a_iStReg)
6049	/** Stores FPU result in a stack register and sets the FPUDP. */
6050	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
6051	iemFpuStoreResultWithMemOp(pIemCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
6052	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
6053	* stack. */
6054	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
6055	iemFpuStoreResultWithMemOpThenPop(pIemCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
6056
6057	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
6058	#define IEM_MC_UPDATE_FSW(a_u16FSW) \
6059	iemFpuUpdateFSW(pIemCpu, a_u16FSW)
6060	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
6061	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
6062	iemFpuUpdateFSWWithMemOp(pIemCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
6063	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
6064	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
6065	iemFpuUpdateFSWThenPop(pIemCpu, a_u16FSW)
6066	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
6067	* stack. */
6068	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
6069	iemFpuUpdateFSWWithMemOpThenPop(pIemCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
6070
6071	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
6072	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
6073	iemFpuStackUnderflow(pIemCpu, a_iStDst)
6074	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
6075	* stack. */
6076	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
6077	iemFpuStackUnderflowThenPop(pIemCpu, a_iStDst)
6078	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
6079	* FPUDS. */
6080	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
6081	iemFpuStackUnderflowWithMemOp(pIemCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
6082	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
6083	* FPUDS. Pops stack. */
6084	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
6085	iemFpuStackUnderflowWithMemOpThenPop(pIemCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
6086
6087	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
6088	* FPUIP, FPUCS and FOP. */
6089	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
6090	iemFpuStackPushOverflow(pIemCpu, a_iStDst)
6091	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
6092	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
6093	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
6094	iemFpuStackPushOverflowWithMemOp(pIemCpu, a_iEffSeg, a_GCPtrEff)
6095
6096
6097	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) {
6098	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit))) {
6099	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBits)) {
6100	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBits))) {
6101	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
6102	if ( !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6103	!= !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6104	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
6105	if ( !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6106	== !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6107	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
6108	if ( (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) \
6109	\|\| !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6110	!= !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6111	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
6112	if ( !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) \
6113	&& !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6114	== !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6115	#define IEM_MC_IF_CX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->cx != 0) {
6116	#define IEM_MC_IF_ECX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->ecx != 0) {
6117	#define IEM_MC_IF_RCX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->rcx != 0) {
6118	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6119	if ( pIemCpu->CTX_SUFF(pCtx)->cx != 0 \
6120	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6121	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6122	if ( pIemCpu->CTX_SUFF(pCtx)->ecx != 0 \
6123	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6124	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6125	if ( pIemCpu->CTX_SUFF(pCtx)->rcx != 0 \
6126	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6127	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6128	if ( pIemCpu->CTX_SUFF(pCtx)->cx != 0 \
6129	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6130	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6131	if ( pIemCpu->CTX_SUFF(pCtx)->ecx != 0 \
6132	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6133	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6134	if ( pIemCpu->CTX_SUFF(pCtx)->rcx != 0 \
6135	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6136	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
6137	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if ((uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
6138	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
6139	if (iemFpuStRegNotEmpty(pIemCpu, (a_iSt)) == VINF_SUCCESS) {
6140	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
6141	if (iemFpuStRegNotEmpty(pIemCpu, (a_iSt)) != VINF_SUCCESS) {
6142	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
6143	if (iemFpuStRegNotEmptyRef(pIemCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
6144	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
6145	if (iemFpu2StRegsNotEmptyRef(pIemCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
6146
6147	#define IEM_MC_ELSE() } else {
6148	#define IEM_MC_ENDIF() } do {} while (0)
6149
6150	/** @} */
6151
6152
6153	/** @name Opcode Debug Helpers.
6154	* @{
6155	*/
6156	#ifdef DEBUG
6157	# define IEMOP_MNEMONIC(a_szMnemonic) \
6158	Log4(("decode - %04x:%RGv %s%s [#%u]\n", pIemCpu->CTX_SUFF(pCtx)->cs, pIemCpu->CTX_SUFF(pCtx)->rip, \
6159	pIemCpu->fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, pIemCpu->cInstructions))
6160	# define IEMOP_MNEMONIC2(a_szMnemonic, a_szOps) \
6161	Log4(("decode - %04x:%RGv %s%s %s [#%u]\n", pIemCpu->CTX_SUFF(pCtx)->cs, pIemCpu->CTX_SUFF(pCtx)->rip, \
6162	pIemCpu->fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, a_szOps, pIemCpu->cInstructions))
6163	#else
6164	# define IEMOP_MNEMONIC(a_szMnemonic) do { } while (0)
6165	# define IEMOP_MNEMONIC2(a_szMnemonic, a_szOps) do { } while (0)
6166	#endif
6167
6168	/** @} */
6169
6170
6171	/** @name Opcode Helpers.
6172	* @{
6173	*/
6174
6175	/** The instruction allows no lock prefixing (in this encoding), throw #UD if
6176	* lock prefixed.
6177	* @deprecated IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX */
6178	#define IEMOP_HLP_NO_LOCK_PREFIX() \
6179	do \
6180	{ \
6181	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
6182	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
6183	} while (0)
6184
6185	/** The instruction is not available in 64-bit mode, throw #UD if we're in
6186	* 64-bit mode. */
6187	#define IEMOP_HLP_NO_64BIT() \
6188	do \
6189	{ \
6190	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
6191	return IEMOP_RAISE_INVALID_OPCODE(); \
6192	} while (0)
6193
6194	/** The instruction defaults to 64-bit operand size if 64-bit mode. */
6195	#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE() \
6196	do \
6197	{ \
6198	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
6199	iemRecalEffOpSize64Default(pIemCpu); \
6200	} while (0)
6201
6202	/**
6203	* Done decoding.
6204	*/
6205	#define IEMOP_HLP_DONE_DECODING() \
6206	do \
6207	{ \
6208	/nothing for now, maybe later... / \
6209	} while (0)
6210
6211	/**
6212	* Done decoding, raise \#UD exception if lock prefix present.
6213	*/
6214	#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX() \
6215	do \
6216	{ \
6217	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
6218	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
6219	} while (0)
6220
6221
6222	/**
6223	* Calculates the effective address of a ModR/M memory operand.
6224	*
6225	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
6226	*
6227	* @return Strict VBox status code.
6228	* @param pIemCpu The IEM per CPU data.
6229	* @param bRm The ModRM byte.
6230	* @param pGCPtrEff Where to return the effective address.
6231	*/
6232	static VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PIEMCPU pIemCpu, uint8_t bRm, PRTGCPTR pGCPtrEff)
6233	{
6234	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
6235	PCCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
6236	#define SET_SS_DEF() \
6237	do \
6238	{ \
6239	if (!(pIemCpu->fPrefixes & IEM_OP_PRF_SEG_MASK)) \
6240	pIemCpu->iEffSeg = X86_SREG_SS; \
6241	} while (0)
6242
6243	/** @todo Check the effective address size crap! */
6244	switch (pIemCpu->enmEffAddrMode)
6245	{
6246	case IEMMODE_16BIT:
6247	{
6248	uint16_t u16EffAddr;
6249
6250	/* Handle the disp16 form with no registers first. */
6251	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
6252	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
6253	else
6254	{
6255	/* Get the displacment. */
6256	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6257	{
6258	case 0: u16EffAddr = 0; break;
6259	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
6260	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
6261	default: AssertFailedReturn(VERR_INTERNAL_ERROR_2); /* (caller checked for these) */
6262	}
6263
6264	/* Add the base and index registers to the disp. */
6265	switch (bRm & X86_MODRM_RM_MASK)
6266	{
6267	case 0: u16EffAddr += pCtx->bx + pCtx->si; break;
6268	case 1: u16EffAddr += pCtx->bx + pCtx->di; break;
6269	case 2: u16EffAddr += pCtx->bp + pCtx->si; SET_SS_DEF(); break;
6270	case 3: u16EffAddr += pCtx->bp + pCtx->di; SET_SS_DEF(); break;
6271	case 4: u16EffAddr += pCtx->si; break;
6272	case 5: u16EffAddr += pCtx->di; break;
6273	case 6: u16EffAddr += pCtx->bp; SET_SS_DEF(); break;
6274	case 7: u16EffAddr += pCtx->bx; break;
6275	}
6276	}
6277
6278	*pGCPtrEff = u16EffAddr;
6279	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#06RGv\n", *pGCPtrEff));
6280	return VINF_SUCCESS;
6281	}
6282
6283	case IEMMODE_32BIT:
6284	{
6285	uint32_t u32EffAddr;
6286
6287	/* Handle the disp32 form with no registers first. */
6288	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
6289	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
6290	else
6291	{
6292	/* Get the register (or SIB) value. */
6293	switch ((bRm & X86_MODRM_RM_MASK))
6294	{
6295	case 0: u32EffAddr = pCtx->eax; break;
6296	case 1: u32EffAddr = pCtx->ecx; break;
6297	case 2: u32EffAddr = pCtx->edx; break;
6298	case 3: u32EffAddr = pCtx->ebx; break;
6299	case 4: /* SIB */
6300	{
6301	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
6302
6303	/* Get the index and scale it. */
6304	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
6305	{
6306	case 0: u32EffAddr = pCtx->eax; break;
6307	case 1: u32EffAddr = pCtx->ecx; break;
6308	case 2: u32EffAddr = pCtx->edx; break;
6309	case 3: u32EffAddr = pCtx->ebx; break;
6310	case 4: u32EffAddr = 0; /none / break;
6311	case 5: u32EffAddr = pCtx->ebp; break;
6312	case 6: u32EffAddr = pCtx->esi; break;
6313	case 7: u32EffAddr = pCtx->edi; break;
6314	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6315	}
6316	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
6317
6318	/* add base */
6319	switch (bSib & X86_SIB_BASE_MASK)
6320	{
6321	case 0: u32EffAddr += pCtx->eax; break;
6322	case 1: u32EffAddr += pCtx->ecx; break;
6323	case 2: u32EffAddr += pCtx->edx; break;
6324	case 3: u32EffAddr += pCtx->ebx; break;
6325	case 4: u32EffAddr += pCtx->esp; SET_SS_DEF(); break;
6326	case 5:
6327	if ((bRm & X86_MODRM_MOD_MASK) != 0)
6328	{
6329	u32EffAddr += pCtx->ebp;
6330	SET_SS_DEF();
6331	}
6332	else
6333	{
6334	uint32_t u32Disp;
6335	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6336	u32EffAddr += u32Disp;
6337	}
6338	break;
6339	case 6: u32EffAddr += pCtx->esi; break;
6340	case 7: u32EffAddr += pCtx->edi; break;
6341	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6342	}
6343	break;
6344	}
6345	case 5: u32EffAddr = pCtx->ebp; SET_SS_DEF(); break;
6346	case 6: u32EffAddr = pCtx->esi; break;
6347	case 7: u32EffAddr = pCtx->edi; break;
6348	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6349	}
6350
6351	/* Get and add the displacement. */
6352	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6353	{
6354	case 0:
6355	break;
6356	case 1:
6357	{
6358	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
6359	u32EffAddr += i8Disp;
6360	break;
6361	}
6362	case 2:
6363	{
6364	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6365	u32EffAddr += u32Disp;
6366	break;
6367	}
6368	default:
6369	AssertFailedReturn(VERR_INTERNAL_ERROR_2); /* (caller checked for these) */
6370	}
6371
6372	}
6373	if (pIemCpu->enmEffAddrMode == IEMMODE_32BIT)
6374	*pGCPtrEff = u32EffAddr;
6375	else
6376	{
6377	Assert(pIemCpu->enmEffAddrMode == IEMMODE_16BIT);
6378	*pGCPtrEff = u32EffAddr & UINT16_MAX;
6379	}
6380	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
6381	return VINF_SUCCESS;
6382	}
6383
6384	case IEMMODE_64BIT:
6385	{
6386	uint64_t u64EffAddr;
6387
6388	/* Handle the rip+disp32 form with no registers first. */
6389	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
6390	{
6391	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
6392	u64EffAddr += pCtx->rip + pIemCpu->offOpcode;
6393	}
6394	else
6395	{
6396	/* Get the register (or SIB) value. */
6397	switch ((bRm & X86_MODRM_RM_MASK) \| pIemCpu->uRexB)
6398	{
6399	case 0: u64EffAddr = pCtx->rax; break;
6400	case 1: u64EffAddr = pCtx->rcx; break;
6401	case 2: u64EffAddr = pCtx->rdx; break;
6402	case 3: u64EffAddr = pCtx->rbx; break;
6403	case 5: u64EffAddr = pCtx->rbp; SET_SS_DEF(); break;
6404	case 6: u64EffAddr = pCtx->rsi; break;
6405	case 7: u64EffAddr = pCtx->rdi; break;
6406	case 8: u64EffAddr = pCtx->r8; break;
6407	case 9: u64EffAddr = pCtx->r9; break;
6408	case 10: u64EffAddr = pCtx->r10; break;
6409	case 11: u64EffAddr = pCtx->r11; break;
6410	case 13: u64EffAddr = pCtx->r13; break;
6411	case 14: u64EffAddr = pCtx->r14; break;
6412	case 15: u64EffAddr = pCtx->r15; break;
6413	/* SIB */
6414	case 4:
6415	case 12:
6416	{
6417	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
6418
6419	/* Get the index and scale it. */
6420	switch (((bSib & X86_SIB_INDEX_SHIFT) >> X86_SIB_INDEX_SMASK) \| pIemCpu->uRexIndex)
6421	{
6422	case 0: u64EffAddr = pCtx->rax; break;
6423	case 1: u64EffAddr = pCtx->rcx; break;
6424	case 2: u64EffAddr = pCtx->rdx; break;
6425	case 3: u64EffAddr = pCtx->rbx; break;
6426	case 4: u64EffAddr = 0; /none / break;
6427	case 5: u64EffAddr = pCtx->rbp; break;
6428	case 6: u64EffAddr = pCtx->rsi; break;
6429	case 7: u64EffAddr = pCtx->rdi; break;
6430	case 8: u64EffAddr = pCtx->r8; break;
6431	case 9: u64EffAddr = pCtx->r9; break;
6432	case 10: u64EffAddr = pCtx->r10; break;
6433	case 11: u64EffAddr = pCtx->r11; break;
6434	case 12: u64EffAddr = pCtx->r12; break;
6435	case 13: u64EffAddr = pCtx->r13; break;
6436	case 14: u64EffAddr = pCtx->r14; break;
6437	case 15: u64EffAddr = pCtx->r15; break;
6438	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6439	}
6440	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
6441
6442	/* add base */
6443	switch ((bSib & X86_SIB_BASE_MASK) \| pIemCpu->uRexB)
6444	{
6445	case 0: u64EffAddr += pCtx->rax; break;
6446	case 1: u64EffAddr += pCtx->rcx; break;
6447	case 2: u64EffAddr += pCtx->rdx; break;
6448	case 3: u64EffAddr += pCtx->rbx; break;
6449	case 4: u64EffAddr += pCtx->rsp; SET_SS_DEF(); break;
6450	case 6: u64EffAddr += pCtx->rsi; break;
6451	case 7: u64EffAddr += pCtx->rdi; break;
6452	case 8: u64EffAddr += pCtx->r8; break;
6453	case 9: u64EffAddr += pCtx->r9; break;
6454	case 10: u64EffAddr += pCtx->r10; break;
6455	case 11: u64EffAddr += pCtx->r11; break;
6456	case 14: u64EffAddr += pCtx->r14; break;
6457	case 15: u64EffAddr += pCtx->r15; break;
6458	/* complicated encodings */
6459	case 5:
6460	case 13:
6461	if ((bRm & X86_MODRM_MOD_MASK) != 0)
6462	{
6463	if (!pIemCpu->uRexB)
6464	{
6465	u64EffAddr += pCtx->rbp;
6466	SET_SS_DEF();
6467	}
6468	else
6469	u64EffAddr += pCtx->r13;
6470	}
6471	else
6472	{
6473	uint32_t u32Disp;
6474	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6475	u64EffAddr += (int32_t)u32Disp;
6476	}
6477	break;
6478	}
6479	break;
6480	}
6481	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6482	}
6483
6484	/* Get and add the displacement. */
6485	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6486	{
6487	case 0:
6488	break;
6489	case 1:
6490	{
6491	int8_t i8Disp;
6492	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
6493	u64EffAddr += i8Disp;
6494	break;
6495	}
6496	case 2:
6497	{
6498	uint32_t u32Disp;
6499	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6500	u64EffAddr += (int32_t)u32Disp;
6501	break;
6502	}
6503	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
6504	}
6505
6506	}
6507	if (pIemCpu->enmEffAddrMode == IEMMODE_64BIT)
6508	*pGCPtrEff = u64EffAddr;
6509	else
6510	*pGCPtrEff = u64EffAddr & UINT16_MAX;
6511	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
6512	return VINF_SUCCESS;
6513	}
6514	}
6515
6516	AssertFailedReturn(VERR_INTERNAL_ERROR_3);
6517	}
6518
6519	/** @} */
6520
6521
6522
6523	/*
6524	* Include the instructions
6525	*/
6526	#include "IEMAllInstructions.cpp.h"
6527
6528
6529
6530
6531	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
6532
6533	/**
6534	* Sets up execution verification mode.
6535	*/
6536	static void iemExecVerificationModeSetup(PIEMCPU pIemCpu)
6537	{
6538	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
6539	PCPUMCTX pOrgCtx = pIemCpu->CTX_SUFF(pCtx);
6540
6541	/*
6542	* Enable verification and/or logging.
6543	*/
6544	pIemCpu->fNoRem = !LogIs6Enabled(); /* logging triggers the no-rem/rem verification stuff */
6545	if ( pIemCpu->fNoRem
6546	#if 0 /* auto enable on first paged protected mode interrupt */
6547	&& pOrgCtx->eflags.Bits.u1IF
6548	&& (pOrgCtx->cr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG)
6549	&& TRPMHasTrap(pVCpu)
6550	&& EMGetInhibitInterruptsPC(pVCpu) != pOrgCtx->rip)
6551	#endif
6552	#if 0
6553	&& pOrgCtx->cs == 0x10
6554	&& ( pOrgCtx->rip == 0x90119e3e
6555	\|\| pOrgCtx->rip == 0x901d9810
6556	)
6557	#endif
6558	#if 0 /* Auto enable DSL - FPU stuff. */
6559	&& pOrgCtx->cs == 0x10
6560	&& (// pOrgCtx->rip == 0xc02ec07f
6561	//\|\| pOrgCtx->rip == 0xc02ec082
6562	//\|\| pOrgCtx->rip == 0xc02ec0c9
6563	0
6564	\|\| pOrgCtx->rip == 0x0c010e7c4 /* fxsave */
6565	)
6566	#endif
6567	#if 1 /* Auto enable DSL - fstp st0 stuff. */
6568	&& pOrgCtx->cs == 0x23
6569	&& pOrgCtx->rip == 0x804aff7
6570	#endif
6571	#if 0
6572	&& pOrgCtx->rip == 0x9022bb3a
6573	#endif
6574	#if 0
6575	&& 0
6576	#endif
6577	)
6578	{
6579	RTLogGroupSettings(NULL, "iem.eo.l6.l2");
6580	RTLogFlags(NULL, "enabled");
6581	pIemCpu->fNoRem = false;
6582	}
6583
6584	/*
6585	* Switch state.
6586	*/
6587	if (IEM_VERIFICATION_ENABLED(pIemCpu))
6588	{
6589	static CPUMCTX s_DebugCtx; /* Ugly! */
6590
6591	s_DebugCtx = *pOrgCtx;
6592	pIemCpu->CTX_SUFF(pCtx) = &s_DebugCtx;
6593	}
6594
6595	/*
6596	* See if there is an interrupt pending in TRPM and inject it if we can.
6597	*/
6598	if ( pOrgCtx->eflags.Bits.u1IF
6599	&& TRPMHasTrap(pVCpu)
6600	&& EMGetInhibitInterruptsPC(pVCpu) != pOrgCtx->rip)
6601	{
6602	uint8_t u8TrapNo;
6603	TRPMEVENT enmType;
6604	RTGCUINT uErrCode;
6605	RTGCPTR uCr2;
6606	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2); AssertRC(rc2);
6607	IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2);
6608	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
6609	TRPMResetTrap(pVCpu);
6610	}
6611
6612	/*
6613	* Reset the counters.
6614	*/
6615	pIemCpu->cIOReads = 0;
6616	pIemCpu->cIOWrites = 0;
6617	pIemCpu->fUndefinedEFlags = 0;
6618
6619	if (IEM_VERIFICATION_ENABLED(pIemCpu))
6620	{
6621	/*
6622	* Free all verification records.
6623	*/
6624	PIEMVERIFYEVTREC pEvtRec = pIemCpu->pIemEvtRecHead;
6625	pIemCpu->pIemEvtRecHead = NULL;
6626	pIemCpu->ppIemEvtRecNext = &pIemCpu->pIemEvtRecHead;
6627	do
6628	{
6629	while (pEvtRec)
6630	{
6631	PIEMVERIFYEVTREC pNext = pEvtRec->pNext;
6632	pEvtRec->pNext = pIemCpu->pFreeEvtRec;
6633	pIemCpu->pFreeEvtRec = pEvtRec;
6634	pEvtRec = pNext;
6635	}
6636	pEvtRec = pIemCpu->pOtherEvtRecHead;
6637	pIemCpu->pOtherEvtRecHead = NULL;
6638	pIemCpu->ppOtherEvtRecNext = &pIemCpu->pOtherEvtRecHead;
6639	} while (pEvtRec);
6640	}
6641	}
6642
6643
6644	/**
6645	* Allocate an event record.
6646	* @returns Poitner to a record.
6647	*/
6648	static PIEMVERIFYEVTREC iemVerifyAllocRecord(PIEMCPU pIemCpu)
6649	{
6650	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
6651	return NULL;
6652
6653	PIEMVERIFYEVTREC pEvtRec = pIemCpu->pFreeEvtRec;
6654	if (pEvtRec)
6655	pIemCpu->pFreeEvtRec = pEvtRec->pNext;
6656	else
6657	{
6658	if (!pIemCpu->ppIemEvtRecNext)
6659	return NULL; /* Too early (fake PCIBIOS), ignore notification. */
6660
6661	pEvtRec = (PIEMVERIFYEVTREC)MMR3HeapAlloc(IEMCPU_TO_VM(pIemCpu), MM_TAG_EM /* lazy bird/, sizeof(pEvtRec));
6662	if (!pEvtRec)
6663	return NULL;
6664	}
6665	pEvtRec->enmEvent = IEMVERIFYEVENT_INVALID;
6666	pEvtRec->pNext = NULL;
6667	return pEvtRec;
6668	}
6669
6670
6671	/**
6672	* IOMMMIORead notification.
6673	*/
6674	VMM_INT_DECL(void) IEMNotifyMMIORead(PVM pVM, RTGCPHYS GCPhys, size_t cbValue)
6675	{
6676	PVMCPU pVCpu = VMMGetCpu(pVM);
6677	if (!pVCpu)
6678	return;
6679	PIEMCPU pIemCpu = &pVCpu->iem.s;
6680	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6681	if (!pEvtRec)
6682	return;
6683	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
6684	pEvtRec->u.RamRead.GCPhys = GCPhys;
6685	pEvtRec->u.RamRead.cb = (uint32_t)cbValue;
6686	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
6687	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
6688	}
6689
6690
6691	/**
6692	* IOMMMIOWrite notification.
6693	*/
6694	VMM_INT_DECL(void) IEMNotifyMMIOWrite(PVM pVM, RTGCPHYS GCPhys, uint32_t u32Value, size_t cbValue)
6695	{
6696	PVMCPU pVCpu = VMMGetCpu(pVM);
6697	if (!pVCpu)
6698	return;
6699	PIEMCPU pIemCpu = &pVCpu->iem.s;
6700	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6701	if (!pEvtRec)
6702	return;
6703	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
6704	pEvtRec->u.RamWrite.GCPhys = GCPhys;
6705	pEvtRec->u.RamWrite.cb = (uint32_t)cbValue;
6706	pEvtRec->u.RamWrite.ab[0] = RT_BYTE1(u32Value);
6707	pEvtRec->u.RamWrite.ab[1] = RT_BYTE2(u32Value);
6708	pEvtRec->u.RamWrite.ab[2] = RT_BYTE3(u32Value);
6709	pEvtRec->u.RamWrite.ab[3] = RT_BYTE4(u32Value);
6710	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
6711	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
6712	}
6713
6714
6715	/**
6716	* IOMIOPortRead notification.
6717	*/
6718	VMM_INT_DECL(void) IEMNotifyIOPortRead(PVM pVM, RTIOPORT Port, size_t cbValue)
6719	{
6720	PVMCPU pVCpu = VMMGetCpu(pVM);
6721	if (!pVCpu)
6722	return;
6723	PIEMCPU pIemCpu = &pVCpu->iem.s;
6724	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6725	if (!pEvtRec)
6726	return;
6727	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_READ;
6728	pEvtRec->u.IOPortRead.Port = Port;
6729	pEvtRec->u.IOPortRead.cbValue = (uint32_t)cbValue;
6730	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
6731	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
6732	}
6733
6734	/**
6735	* IOMIOPortWrite notification.
6736	*/
6737	VMM_INT_DECL(void) IEMNotifyIOPortWrite(PVM pVM, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
6738	{
6739	PVMCPU pVCpu = VMMGetCpu(pVM);
6740	if (!pVCpu)
6741	return;
6742	PIEMCPU pIemCpu = &pVCpu->iem.s;
6743	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6744	if (!pEvtRec)
6745	return;
6746	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_WRITE;
6747	pEvtRec->u.IOPortWrite.Port = Port;
6748	pEvtRec->u.IOPortWrite.cbValue = (uint32_t)cbValue;
6749	pEvtRec->u.IOPortWrite.u32Value = u32Value;
6750	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
6751	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
6752	}
6753
6754
6755	VMM_INT_DECL(void) IEMNotifyIOPortReadString(PVM pVM, RTIOPORT Port, RTGCPTR GCPtrDst, RTGCUINTREG cTransfers, size_t cbValue)
6756	{
6757	AssertFailed();
6758	}
6759
6760
6761	VMM_INT_DECL(void) IEMNotifyIOPortWriteString(PVM pVM, RTIOPORT Port, RTGCPTR GCPtrSrc, RTGCUINTREG cTransfers, size_t cbValue)
6762	{
6763	AssertFailed();
6764	}
6765
6766
6767	/**
6768	* Fakes and records an I/O port read.
6769	*
6770	* @returns VINF_SUCCESS.
6771	* @param pIemCpu The IEM per CPU data.
6772	* @param Port The I/O port.
6773	* @param pu32Value Where to store the fake value.
6774	* @param cbValue The size of the access.
6775	*/
6776	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
6777	{
6778	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6779	if (pEvtRec)
6780	{
6781	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_READ;
6782	pEvtRec->u.IOPortRead.Port = Port;
6783	pEvtRec->u.IOPortRead.cbValue = (uint32_t)cbValue;
6784	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
6785	*pIemCpu->ppIemEvtRecNext = pEvtRec;
6786	}
6787	pIemCpu->cIOReads++;
6788	*pu32Value = 0xcccccccc;
6789	return VINF_SUCCESS;
6790	}
6791
6792
6793	/**
6794	* Fakes and records an I/O port write.
6795	*
6796	* @returns VINF_SUCCESS.
6797	* @param pIemCpu The IEM per CPU data.
6798	* @param Port The I/O port.
6799	* @param u32Value The value being written.
6800	* @param cbValue The size of the access.
6801	*/
6802	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
6803	{
6804	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6805	if (pEvtRec)
6806	{
6807	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_WRITE;
6808	pEvtRec->u.IOPortWrite.Port = Port;
6809	pEvtRec->u.IOPortWrite.cbValue = (uint32_t)cbValue;
6810	pEvtRec->u.IOPortWrite.u32Value = u32Value;
6811	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
6812	*pIemCpu->ppIemEvtRecNext = pEvtRec;
6813	}
6814	pIemCpu->cIOWrites++;
6815	return VINF_SUCCESS;
6816	}
6817
6818
6819	/**
6820	* Used to add extra details about a stub case.
6821	* @param pIemCpu The IEM per CPU state.
6822	*/
6823	static void iemVerifyAssertMsg2(PIEMCPU pIemCpu)
6824	{
6825	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
6826	PVM pVM = IEMCPU_TO_VM(pIemCpu);
6827	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
6828	char szRegs[4096];
6829	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
6830	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
6831	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
6832	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
6833	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
6834	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
6835	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
6836	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
6837	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
6838	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
6839	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
6840	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
6841	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
6842	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
6843	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
6844	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
6845	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
6846	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
6847	" efer=%016VR{efer}\n"
6848	" pat=%016VR{pat}\n"
6849	" sf_mask=%016VR{sf_mask}\n"
6850	"krnl_gs_base=%016VR{krnl_gs_base}\n"
6851	" lstar=%016VR{lstar}\n"
6852	" star=%016VR{star} cstar=%016VR{cstar}\n"
6853	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
6854	);
6855
6856	char szInstr1[256];
6857	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip - pIemCpu->offOpcode,
6858	DBGF_DISAS_FLAGS_DEFAULT_MODE,
6859	szInstr1, sizeof(szInstr1), NULL);
6860	char szInstr2[256];
6861	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
6862	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
6863	szInstr2, sizeof(szInstr2), NULL);
6864
6865	RTAssertMsg2Weak("%s%s\n%s\n", szRegs, szInstr1, szInstr2);
6866	}
6867
6868
6869	/**
6870	* Used by iemVerifyAssertRecord and iemVerifyAssertRecords to add a record
6871	* dump to the assertion info.
6872	*
6873	* @param pEvtRec The record to dump.
6874	*/
6875	static void iemVerifyAssertAddRecordDump(PIEMVERIFYEVTREC pEvtRec)
6876	{
6877	switch (pEvtRec->enmEvent)
6878	{
6879	case IEMVERIFYEVENT_IOPORT_READ:
6880	RTAssertMsg2Add("I/O PORT READ from %#6x, %d bytes\n",
6881	pEvtRec->u.IOPortWrite.Port,
6882	pEvtRec->u.IOPortWrite.cbValue);
6883	break;
6884	case IEMVERIFYEVENT_IOPORT_WRITE:
6885	RTAssertMsg2Add("I/O PORT WRITE to %#6x, %d bytes, value %#x\n",
6886	pEvtRec->u.IOPortWrite.Port,
6887	pEvtRec->u.IOPortWrite.cbValue,
6888	pEvtRec->u.IOPortWrite.u32Value);
6889	break;
6890	case IEMVERIFYEVENT_RAM_READ:
6891	RTAssertMsg2Add("RAM READ at %RGp, %#4zx bytes\n",
6892	pEvtRec->u.RamRead.GCPhys,
6893	pEvtRec->u.RamRead.cb);
6894	break;
6895	case IEMVERIFYEVENT_RAM_WRITE:
6896	RTAssertMsg2Add("RAM WRITE at %RGp, %#4zx bytes: %.*Rhxs\n",
6897	pEvtRec->u.RamWrite.GCPhys,
6898	pEvtRec->u.RamWrite.cb,
6899	(int)pEvtRec->u.RamWrite.cb,
6900	pEvtRec->u.RamWrite.ab);
6901	break;
6902	default:
6903	AssertMsgFailed(("Invalid event type %d\n", pEvtRec->enmEvent));
6904	break;
6905	}
6906	}
6907
6908
6909	/**
6910	* Raises an assertion on the specified record, showing the given message with
6911	* a record dump attached.
6912	*
6913	* @param pIemCpu The IEM per CPU data.
6914	* @param pEvtRec1 The first record.
6915	* @param pEvtRec2 The second record.
6916	* @param pszMsg The message explaining why we're asserting.
6917	*/
6918	static void iemVerifyAssertRecords(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec1, PIEMVERIFYEVTREC pEvtRec2, const char *pszMsg)
6919	{
6920	RTAssertMsg1(pszMsg, __LINE__, __FILE__, __PRETTY_FUNCTION__);
6921	iemVerifyAssertAddRecordDump(pEvtRec1);
6922	iemVerifyAssertAddRecordDump(pEvtRec2);
6923	iemVerifyAssertMsg2(pIemCpu);
6924	RTAssertPanic();
6925	}
6926
6927
6928	/**
6929	* Raises an assertion on the specified record, showing the given message with
6930	* a record dump attached.
6931	*
6932	* @param pIemCpu The IEM per CPU data.
6933	* @param pEvtRec1 The first record.
6934	* @param pszMsg The message explaining why we're asserting.
6935	*/
6936	static void iemVerifyAssertRecord(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec, const char *pszMsg)
6937	{
6938	RTAssertMsg1(pszMsg, __LINE__, __FILE__, __PRETTY_FUNCTION__);
6939	iemVerifyAssertAddRecordDump(pEvtRec);
6940	iemVerifyAssertMsg2(pIemCpu);
6941	RTAssertPanic();
6942	}
6943
6944
6945	/**
6946	* Verifies a write record.
6947	*
6948	* @param pIemCpu The IEM per CPU data.
6949	* @param pEvtRec The write record.
6950	*/
6951	static void iemVerifyWriteRecord(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec)
6952	{
6953	uint8_t abBuf[sizeof(pEvtRec->u.RamWrite.ab)]; RT_ZERO(abBuf);
6954	Assert(sizeof(abBuf) >= pEvtRec->u.RamWrite.cb);
6955	int rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), abBuf, pEvtRec->u.RamWrite.GCPhys, pEvtRec->u.RamWrite.cb);
6956	if ( RT_FAILURE(rc)
6957	\|\| memcmp(abBuf, pEvtRec->u.RamWrite.ab, pEvtRec->u.RamWrite.cb) )
6958	{
6959	/* fend off ins */
6960	if ( !pIemCpu->cIOReads
6961	\|\| pEvtRec->u.RamWrite.ab[0] != 0xcc
6962	\|\| ( pEvtRec->u.RamWrite.cb != 1
6963	&& pEvtRec->u.RamWrite.cb != 2
6964	&& pEvtRec->u.RamWrite.cb != 4) )
6965	{
6966	/* fend off ROMs */
6967	if ( pEvtRec->u.RamWrite.GCPhys - UINT32_C(0x000c0000) > UINT32_C(0x8000)
6968	&& pEvtRec->u.RamWrite.GCPhys - UINT32_C(0x000e0000) > UINT32_C(0x20000)
6969	&& pEvtRec->u.RamWrite.GCPhys - UINT32_C(0xfffc0000) > UINT32_C(0x40000) )
6970	{
6971	/* fend off fxsave */
6972	if (pEvtRec->u.RamWrite.cb != 512)
6973	{
6974	RTAssertMsg1(NULL, __LINE__, __FILE__, __PRETTY_FUNCTION__);
6975	RTAssertMsg2Weak("Memory at %RGv differs\n", pEvtRec->u.RamWrite.GCPhys);
6976	RTAssertMsg2Add("REM: %.*Rhxs\n"
6977	"IEM: %.*Rhxs\n",
6978	pEvtRec->u.RamWrite.cb, abBuf,
6979	pEvtRec->u.RamWrite.cb, pEvtRec->u.RamWrite.ab);
6980	iemVerifyAssertAddRecordDump(pEvtRec);
6981	iemVerifyAssertMsg2(pIemCpu);
6982	RTAssertPanic();
6983	}
6984	}
6985	}
6986	}
6987
6988	}
6989
6990	/**
6991	* Performs the post-execution verfication checks.
6992	*/
6993	static void iemExecVerificationModeCheck(PIEMCPU pIemCpu)
6994	{
6995	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
6996	return;
6997
6998	/*
6999	* Switch back the state.
7000	*/
7001	PCPUMCTX pOrgCtx = CPUMQueryGuestCtxPtr(IEMCPU_TO_VMCPU(pIemCpu));
7002	PCPUMCTX pDebugCtx = pIemCpu->CTX_SUFF(pCtx);
7003	Assert(pOrgCtx != pDebugCtx);
7004	pIemCpu->CTX_SUFF(pCtx) = pOrgCtx;
7005
7006	/*
7007	* Execute the instruction in REM.
7008	*/
7009	PVM pVM = IEMCPU_TO_VM(pIemCpu);
7010	EMRemLock(pVM);
7011	int rc = REMR3EmulateInstruction(pVM, IEMCPU_TO_VMCPU(pIemCpu));
7012	AssertRC(rc);
7013	EMRemUnlock(pVM);
7014
7015	/*
7016	* Compare the register states.
7017	*/
7018	unsigned cDiffs = 0;
7019	if (memcmp(pOrgCtx, pDebugCtx, sizeof(*pDebugCtx)))
7020	{
7021	Log(("REM and IEM ends up with different registers!\n"));
7022
7023	# define CHECK_FIELD(a_Field) \
7024	do \
7025	{ \
7026	if (pOrgCtx->a_Field != pDebugCtx->a_Field) \
7027	{ \
7028	switch (sizeof(pOrgCtx->a_Field)) \
7029	{ \
7030	case 1: RTAssertMsg2Weak(" %8s differs - iem=%02x - rem=%02x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7031	case 2: RTAssertMsg2Weak(" %8s differs - iem=%04x - rem=%04x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7032	case 4: RTAssertMsg2Weak(" %8s differs - iem=%08x - rem=%08x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7033	case 8: RTAssertMsg2Weak(" %8s differs - iem=%016llx - rem=%016llx\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7034	default: RTAssertMsg2Weak(" %8s differs\n", #a_Field); break; \
7035	} \
7036	cDiffs++; \
7037	} \
7038	} while (0)
7039
7040	# define CHECK_BIT_FIELD(a_Field) \
7041	do \
7042	{ \
7043	if (pOrgCtx->a_Field != pDebugCtx->a_Field) \
7044	{ \
7045	RTAssertMsg2Weak(" %8s differs - iem=%02x - rem=%02x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); \
7046	cDiffs++; \
7047	} \
7048	} while (0)
7049
7050	# define CHECK_SEL(a_Sel) \
7051	do \
7052	{ \
7053	CHECK_FIELD(a_Sel); \
7054	if ( pOrgCtx->a_Sel##Hid.Attr.u != pDebugCtx->a_Sel##Hid.Attr.u \
7055	&& (pOrgCtx->a_Sel##Hid.Attr.u \| X86_SEL_TYPE_ACCESSED) != pDebugCtx->a_Sel##Hid.Attr.u) \
7056	{ \
7057	RTAssertMsg2Weak(" %8sHid.Attr differs - iem=%02x - rem=%02x\n", #a_Sel, pDebugCtx->a_Sel##Hid.Attr.u, pOrgCtx->a_Sel##Hid.Attr.u); \
7058	cDiffs++; \
7059	} \
7060	CHECK_FIELD(a_Sel##Hid.u64Base); \
7061	CHECK_FIELD(a_Sel##Hid.u32Limit); \
7062	} while (0)
7063
7064	#if 1 /* The recompiler doesn't update these the intel way. */
7065	pOrgCtx->fpu.FOP = pDebugCtx->fpu.FOP;
7066	pOrgCtx->fpu.FPUIP = pDebugCtx->fpu.FPUIP;
7067	pOrgCtx->fpu.CS = pDebugCtx->fpu.CS;
7068	pOrgCtx->fpu.Rsrvd1 = pDebugCtx->fpu.Rsrvd1;
7069	pOrgCtx->fpu.FPUDP = pDebugCtx->fpu.FPUDP;
7070	pOrgCtx->fpu.DS = pDebugCtx->fpu.DS;
7071	pOrgCtx->fpu.Rsrvd2 = pDebugCtx->fpu.Rsrvd2;
7072	pOrgCtx->fpu.MXCSR_MASK = pDebugCtx->fpu.MXCSR_MASK; /* only for the time being - old snapshots here. */
7073	if ((pOrgCtx->fpu.FSW & X86_FSW_TOP_MASK) == (pDebugCtx->fpu.FSW & X86_FSW_TOP_MASK))
7074	pOrgCtx->fpu.FSW = pDebugCtx->fpu.FSW;
7075	#endif
7076	if (memcmp(&pOrgCtx->fpu, &pDebugCtx->fpu, sizeof(pDebugCtx->fpu)))
7077	{
7078	RTAssertMsg2Weak(" the FPU state differs\n");
7079	cDiffs++;
7080	CHECK_FIELD(fpu.FCW);
7081	CHECK_FIELD(fpu.FSW);
7082	CHECK_FIELD(fpu.FTW);
7083	CHECK_FIELD(fpu.FOP);
7084	CHECK_FIELD(fpu.FPUIP);
7085	CHECK_FIELD(fpu.CS);
7086	CHECK_FIELD(fpu.Rsrvd1);
7087	CHECK_FIELD(fpu.FPUDP);
7088	CHECK_FIELD(fpu.DS);
7089	CHECK_FIELD(fpu.Rsrvd2);
7090	CHECK_FIELD(fpu.MXCSR);
7091	CHECK_FIELD(fpu.MXCSR_MASK);
7092	CHECK_FIELD(fpu.aRegs[0].au64[0]); CHECK_FIELD(fpu.aRegs[0].au64[1]);
7093	CHECK_FIELD(fpu.aRegs[1].au64[0]); CHECK_FIELD(fpu.aRegs[1].au64[1]);
7094	CHECK_FIELD(fpu.aRegs[2].au64[0]); CHECK_FIELD(fpu.aRegs[2].au64[1]);
7095	CHECK_FIELD(fpu.aRegs[3].au64[0]); CHECK_FIELD(fpu.aRegs[3].au64[1]);
7096	CHECK_FIELD(fpu.aRegs[4].au64[0]); CHECK_FIELD(fpu.aRegs[4].au64[1]);
7097	CHECK_FIELD(fpu.aRegs[5].au64[0]); CHECK_FIELD(fpu.aRegs[5].au64[1]);
7098	CHECK_FIELD(fpu.aRegs[6].au64[0]); CHECK_FIELD(fpu.aRegs[6].au64[1]);
7099	CHECK_FIELD(fpu.aRegs[7].au64[0]); CHECK_FIELD(fpu.aRegs[7].au64[1]);
7100	CHECK_FIELD(fpu.aXMM[ 0].au64[0]); CHECK_FIELD(fpu.aXMM[ 0].au64[1]);
7101	CHECK_FIELD(fpu.aXMM[ 1].au64[0]); CHECK_FIELD(fpu.aXMM[ 1].au64[1]);
7102	CHECK_FIELD(fpu.aXMM[ 2].au64[0]); CHECK_FIELD(fpu.aXMM[ 2].au64[1]);
7103	CHECK_FIELD(fpu.aXMM[ 3].au64[0]); CHECK_FIELD(fpu.aXMM[ 3].au64[1]);
7104	CHECK_FIELD(fpu.aXMM[ 4].au64[0]); CHECK_FIELD(fpu.aXMM[ 4].au64[1]);
7105	CHECK_FIELD(fpu.aXMM[ 5].au64[0]); CHECK_FIELD(fpu.aXMM[ 5].au64[1]);
7106	CHECK_FIELD(fpu.aXMM[ 6].au64[0]); CHECK_FIELD(fpu.aXMM[ 6].au64[1]);
7107	CHECK_FIELD(fpu.aXMM[ 7].au64[0]); CHECK_FIELD(fpu.aXMM[ 7].au64[1]);
7108	CHECK_FIELD(fpu.aXMM[ 8].au64[0]); CHECK_FIELD(fpu.aXMM[ 8].au64[1]);
7109	CHECK_FIELD(fpu.aXMM[ 9].au64[0]); CHECK_FIELD(fpu.aXMM[ 9].au64[1]);
7110	CHECK_FIELD(fpu.aXMM[10].au64[0]); CHECK_FIELD(fpu.aXMM[10].au64[1]);
7111	CHECK_FIELD(fpu.aXMM[11].au64[0]); CHECK_FIELD(fpu.aXMM[11].au64[1]);
7112	CHECK_FIELD(fpu.aXMM[12].au64[0]); CHECK_FIELD(fpu.aXMM[12].au64[1]);
7113	CHECK_FIELD(fpu.aXMM[13].au64[0]); CHECK_FIELD(fpu.aXMM[13].au64[1]);
7114	CHECK_FIELD(fpu.aXMM[14].au64[0]); CHECK_FIELD(fpu.aXMM[14].au64[1]);
7115	CHECK_FIELD(fpu.aXMM[15].au64[0]); CHECK_FIELD(fpu.aXMM[15].au64[1]);
7116	for (unsigned i = 0; i < RT_ELEMENTS(pOrgCtx->fpu.au32RsrvdRest); i++)
7117	CHECK_FIELD(fpu.au32RsrvdRest[i]);
7118	}
7119	CHECK_FIELD(rip);
7120	uint32_t fFlagsMask = UINT32_MAX & ~pIemCpu->fUndefinedEFlags;
7121	if ((pOrgCtx->rflags.u & fFlagsMask) != (pDebugCtx->rflags.u & fFlagsMask))
7122	{
7123	RTAssertMsg2Weak(" rflags differs - iem=%08llx rem=%08llx\n", pDebugCtx->rflags.u, pOrgCtx->rflags.u);
7124	CHECK_BIT_FIELD(rflags.Bits.u1CF);
7125	CHECK_BIT_FIELD(rflags.Bits.u1Reserved0);
7126	CHECK_BIT_FIELD(rflags.Bits.u1PF);
7127	CHECK_BIT_FIELD(rflags.Bits.u1Reserved1);
7128	CHECK_BIT_FIELD(rflags.Bits.u1AF);
7129	CHECK_BIT_FIELD(rflags.Bits.u1Reserved2);
7130	CHECK_BIT_FIELD(rflags.Bits.u1ZF);
7131	CHECK_BIT_FIELD(rflags.Bits.u1SF);
7132	CHECK_BIT_FIELD(rflags.Bits.u1TF);
7133	CHECK_BIT_FIELD(rflags.Bits.u1IF);
7134	CHECK_BIT_FIELD(rflags.Bits.u1DF);
7135	CHECK_BIT_FIELD(rflags.Bits.u1OF);
7136	CHECK_BIT_FIELD(rflags.Bits.u2IOPL);
7137	CHECK_BIT_FIELD(rflags.Bits.u1NT);
7138	CHECK_BIT_FIELD(rflags.Bits.u1Reserved3);
7139	CHECK_BIT_FIELD(rflags.Bits.u1RF);
7140	CHECK_BIT_FIELD(rflags.Bits.u1VM);
7141	CHECK_BIT_FIELD(rflags.Bits.u1AC);
7142	CHECK_BIT_FIELD(rflags.Bits.u1VIF);
7143	CHECK_BIT_FIELD(rflags.Bits.u1VIP);
7144	CHECK_BIT_FIELD(rflags.Bits.u1ID);
7145	}
7146
7147	if (pIemCpu->cIOReads != 1 && !pIemCpu->fIgnoreRaxRdx)
7148	CHECK_FIELD(rax);
7149	CHECK_FIELD(rcx);
7150	if (!pIemCpu->fIgnoreRaxRdx)
7151	CHECK_FIELD(rdx);
7152	CHECK_FIELD(rbx);
7153	CHECK_FIELD(rsp);
7154	CHECK_FIELD(rbp);
7155	CHECK_FIELD(rsi);
7156	CHECK_FIELD(rdi);
7157	CHECK_FIELD(r8);
7158	CHECK_FIELD(r9);
7159	CHECK_FIELD(r10);
7160	CHECK_FIELD(r11);
7161	CHECK_FIELD(r12);
7162	CHECK_FIELD(r13);
7163	CHECK_SEL(cs);
7164	CHECK_SEL(ss);
7165	CHECK_SEL(ds);
7166	CHECK_SEL(es);
7167	CHECK_SEL(fs);
7168	CHECK_SEL(gs);
7169	CHECK_FIELD(cr0);
7170	CHECK_FIELD(cr2);
7171	CHECK_FIELD(cr3);
7172	CHECK_FIELD(cr4);
7173	CHECK_FIELD(dr[0]);
7174	CHECK_FIELD(dr[1]);
7175	CHECK_FIELD(dr[2]);
7176	CHECK_FIELD(dr[3]);
7177	CHECK_FIELD(dr[6]);
7178	if ((pOrgCtx->dr[7] & ~X86_DR7_MB1_MASK) != (pDebugCtx->dr[7] & ~X86_DR7_MB1_MASK)) /* REM 'mov drX,greg' bug.*/
7179	CHECK_FIELD(dr[7]);
7180	CHECK_FIELD(gdtr.cbGdt);
7181	CHECK_FIELD(gdtr.pGdt);
7182	CHECK_FIELD(idtr.cbIdt);
7183	CHECK_FIELD(idtr.pIdt);
7184	CHECK_FIELD(ldtr);
7185	CHECK_FIELD(ldtrHid.u64Base);
7186	CHECK_FIELD(ldtrHid.u32Limit);
7187	CHECK_FIELD(ldtrHid.Attr.u);
7188	CHECK_FIELD(tr);
7189	CHECK_FIELD(trHid.u64Base);
7190	CHECK_FIELD(trHid.u32Limit);
7191	CHECK_FIELD(trHid.Attr.u);
7192	CHECK_FIELD(SysEnter.cs);
7193	CHECK_FIELD(SysEnter.eip);
7194	CHECK_FIELD(SysEnter.esp);
7195	CHECK_FIELD(msrEFER);
7196	CHECK_FIELD(msrSTAR);
7197	CHECK_FIELD(msrPAT);
7198	CHECK_FIELD(msrLSTAR);
7199	CHECK_FIELD(msrCSTAR);
7200	CHECK_FIELD(msrSFMASK);
7201	CHECK_FIELD(msrKERNELGSBASE);
7202
7203	if (cDiffs != 0)
7204	{
7205	if (LogIs3Enabled())
7206	DBGFR3Info(pVM, "cpumguest", "verbose", NULL);
7207	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__);
7208	iemVerifyAssertMsg2(pIemCpu);
7209	RTAssertPanic();
7210	}
7211	# undef CHECK_FIELD
7212	# undef CHECK_BIT_FIELD
7213	}
7214
7215	/*
7216	* If the register state compared fine, check the verification event
7217	* records.
7218	*/
7219	if (cDiffs == 0)
7220	{
7221	/*
7222	* Compare verficiation event records.
7223	* - I/O port accesses should be a 1:1 match.
7224	*/
7225	PIEMVERIFYEVTREC pIemRec = pIemCpu->pIemEvtRecHead;
7226	PIEMVERIFYEVTREC pOtherRec = pIemCpu->pOtherEvtRecHead;
7227	while (pIemRec && pOtherRec)
7228	{
7229	/* Since we might miss RAM writes and reads, ignore reads and check
7230	that any written memory is the same extra ones. */
7231	while ( IEMVERIFYEVENT_IS_RAM(pIemRec->enmEvent)
7232	&& !IEMVERIFYEVENT_IS_RAM(pOtherRec->enmEvent)
7233	&& pIemRec->pNext)
7234	{
7235	if (pIemRec->enmEvent == IEMVERIFYEVENT_RAM_WRITE)
7236	iemVerifyWriteRecord(pIemCpu, pIemRec);
7237	pIemRec = pIemRec->pNext;
7238	}
7239
7240	/* Do the compare. */
7241	if (pIemRec->enmEvent != pOtherRec->enmEvent)
7242	{
7243	iemVerifyAssertRecords(pIemCpu, pIemRec, pOtherRec, "Type mismatches");
7244	break;
7245	}
7246	bool fEquals;
7247	switch (pIemRec->enmEvent)
7248	{
7249	case IEMVERIFYEVENT_IOPORT_READ:
7250	fEquals = pIemRec->u.IOPortRead.Port == pOtherRec->u.IOPortRead.Port
7251	&& pIemRec->u.IOPortRead.cbValue == pOtherRec->u.IOPortRead.cbValue;
7252	break;
7253	case IEMVERIFYEVENT_IOPORT_WRITE:
7254	fEquals = pIemRec->u.IOPortWrite.Port == pOtherRec->u.IOPortWrite.Port
7255	&& pIemRec->u.IOPortWrite.cbValue == pOtherRec->u.IOPortWrite.cbValue
7256	&& pIemRec->u.IOPortWrite.u32Value == pOtherRec->u.IOPortWrite.u32Value;
7257	break;
7258	case IEMVERIFYEVENT_RAM_READ:
7259	fEquals = pIemRec->u.RamRead.GCPhys == pOtherRec->u.RamRead.GCPhys
7260	&& pIemRec->u.RamRead.cb == pOtherRec->u.RamRead.cb;
7261	break;
7262	case IEMVERIFYEVENT_RAM_WRITE:
7263	fEquals = pIemRec->u.RamWrite.GCPhys == pOtherRec->u.RamWrite.GCPhys
7264	&& pIemRec->u.RamWrite.cb == pOtherRec->u.RamWrite.cb
7265	&& !memcmp(pIemRec->u.RamWrite.ab, pOtherRec->u.RamWrite.ab, pIemRec->u.RamWrite.cb);
7266	break;
7267	default:
7268	fEquals = false;
7269	break;
7270	}
7271	if (!fEquals)
7272	{
7273	iemVerifyAssertRecords(pIemCpu, pIemRec, pOtherRec, "Mismatch");
7274	break;
7275	}
7276
7277	/* advance */
7278	pIemRec = pIemRec->pNext;
7279	pOtherRec = pOtherRec->pNext;
7280	}
7281
7282	/* Ignore extra writes and reads. */
7283	while (pIemRec && IEMVERIFYEVENT_IS_RAM(pIemRec->enmEvent))
7284	{
7285	if (pIemRec->enmEvent == IEMVERIFYEVENT_RAM_WRITE)
7286	iemVerifyWriteRecord(pIemCpu, pIemRec);
7287	pIemRec = pIemRec->pNext;
7288	}
7289	if (pIemRec != NULL)
7290	iemVerifyAssertRecord(pIemCpu, pIemRec, "Extra IEM record!");
7291	else if (pOtherRec != NULL)
7292	iemVerifyAssertRecord(pIemCpu, pIemRec, "Extra Other record!");
7293	}
7294	pIemCpu->CTX_SUFF(pCtx) = pOrgCtx;
7295
7296	#if 0
7297	/*
7298	* HACK ALERT! You don't normally want to verify a whole boot sequence.
7299	*/
7300	if (pIemCpu->cInstructions == 1)
7301	RTLogFlags(NULL, "disabled");
7302	#endif
7303	}
7304
7305	#else /* !IEM_VERIFICATION_MODE \|\| !IN_RING3 */
7306
7307	/* stubs */
7308	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
7309	{
7310	NOREF(pIemCpu); NOREF(Port); NOREF(pu32Value); NOREF(cbValue);
7311	return VERR_INTERNAL_ERROR;
7312	}
7313
7314	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
7315	{
7316	NOREF(pIemCpu); NOREF(Port); NOREF(u32Value); NOREF(cbValue);
7317	return VERR_INTERNAL_ERROR;
7318	}
7319
7320	#endif /* !IEM_VERIFICATION_MODE \|\| !IN_RING3 */
7321
7322
7323	/**
7324	* Execute one instruction.
7325	*
7326	* @return Strict VBox status code.
7327	* @param pVCpu The current virtual CPU.
7328	*/
7329	VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPU pVCpu)
7330	{
7331	PIEMCPU pIemCpu = &pVCpu->iem.s;
7332
7333	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7334	iemExecVerificationModeSetup(pIemCpu);
7335	#endif
7336	#ifdef LOG_ENABLED
7337	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7338	if (LogIs2Enabled())
7339	{
7340	char szInstr[256];
7341	uint32_t cbInstr = 0;
7342	DBGFR3DisasInstrEx(pVCpu->pVMR3, pVCpu->idCpu, 0, 0,
7343	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
7344	szInstr, sizeof(szInstr), &cbInstr);
7345
7346	Log3(("**** "
7347	" eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
7348	" eip=%08x esp=%08x ebp=%08x iopl=%d\n"
7349	" cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
7350	" fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
7351	" %s\n"
7352	,
7353	pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
7354	pCtx->eip, pCtx->esp, pCtx->ebp, pCtx->eflags.Bits.u2IOPL,
7355	(RTSEL)pCtx->cs, (RTSEL)pCtx->ss, (RTSEL)pCtx->ds, (RTSEL)pCtx->es,
7356	(RTSEL)pCtx->fs, (RTSEL)pCtx->gs, pCtx->eflags.u,
7357	pCtx->fpu.FSW, pCtx->fpu.FCW, pCtx->fpu.FTW, pCtx->fpu.MXCSR, pCtx->fpu.MXCSR_MASK,
7358	szInstr));
7359
7360	if (LogIs3Enabled())
7361	DBGFR3Info(pVCpu->pVMR3, "cpumguest", "verbose", NULL);
7362	}
7363	else
7364	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n",
7365	pCtx->cs, pCtx->rip, pCtx->ss, pCtx->rsp, pCtx->eflags.u));
7366	#endif
7367
7368	/*
7369	* Do the decoding and emulation.
7370	*/
7371	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu);
7372	if (rcStrict != VINF_SUCCESS)
7373	{
7374	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7375	iemExecVerificationModeCheck(pIemCpu);
7376	#endif
7377	return rcStrict;
7378	}
7379
7380	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
7381	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
7382	if (rcStrict == VINF_SUCCESS)
7383	pIemCpu->cInstructions++;
7384	//#ifdef DEBUG
7385	// AssertMsg(pIemCpu->offOpcode == cbInstr \|\| rcStrict != VINF_SUCCESS, ("%u %u\n", pIemCpu->offOpcode, cbInstr));
7386	//#endif
7387
7388	/* Execute the next instruction as well if a cli, pop ss or
7389	mov ss, Gr has just completed successfully. */
7390	if ( rcStrict == VINF_SUCCESS
7391	&& VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
7392	&& EMGetInhibitInterruptsPC(pVCpu) == pIemCpu->CTX_SUFF(pCtx)->rip )
7393	{
7394	rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu);
7395	if (rcStrict == VINF_SUCCESS)
7396	{
7397	b; IEM_OPCODE_GET_NEXT_U8(&b);
7398	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
7399	if (rcStrict == VINF_SUCCESS)
7400	pIemCpu->cInstructions++;
7401	}
7402	EMSetInhibitInterruptsPC(pVCpu, UINT64_C(0x7777555533331111));
7403	}
7404
7405	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7406	/*
7407	* Assert some sanity.
7408	*/
7409	iemExecVerificationModeCheck(pIemCpu);
7410	#endif
7411	if (rcStrict != VINF_SUCCESS)
7412	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
7413	pCtx->cs, pCtx->rip, pCtx->ss, pCtx->rsp, pCtx->eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
7414	return rcStrict;
7415	}
7416
7417
7418	/**
7419	* Injects a trap, fault, abort, software interrupt or external interrupt.
7420	*
7421	* The parameter list matches TRPMQueryTrapAll pretty closely.
7422	*
7423	* @returns Strict VBox status code.
7424	* @param pVCpu The current virtual CPU.
7425	* @param u8TrapNo The trap number.
7426	* @param enmType What type is it (trap/fault/abort), software
7427	* interrupt or hardware interrupt.
7428	* @param uErrCode The error code if applicable.
7429	* @param uCr2 The CR2 value if applicable.
7430	*/
7431	VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPU pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2)
7432	{
7433	iemInitDecoder(&pVCpu->iem.s);
7434
7435	uint32_t fFlags;
7436	switch (enmType)
7437	{
7438	case TRPM_HARDWARE_INT:
7439	LogFlow(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
7440	fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7441	uErrCode = uCr2 = 0;
7442	break;
7443
7444	case TRPM_SOFTWARE_INT:
7445	LogFlow(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
7446	fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7447	uErrCode = uCr2 = 0;
7448	break;
7449
7450	case TRPM_TRAP:
7451	LogFlow(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
7452	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7453	if (u8TrapNo == X86_XCPT_PF)
7454	fFlags \|= IEM_XCPT_FLAGS_CR2;
7455	switch (u8TrapNo)
7456	{
7457	case X86_XCPT_DF:
7458	case X86_XCPT_TS:
7459	case X86_XCPT_NP:
7460	case X86_XCPT_SS:
7461	case X86_XCPT_PF:
7462	case X86_XCPT_AC:
7463	fFlags \|= IEM_XCPT_FLAGS_ERR;
7464	break;
7465	}
7466	break;
7467
7468	IEM_NOT_REACHED_DEFAULT_CASE_RET();
7469	}
7470
7471	return iemRaiseXcptOrInt(&pVCpu->iem.s, 0, u8TrapNo, fFlags, uErrCode, uCr2);
7472	}
7473

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 40209

Download in other formats: