HMVMXR0.cpp@ 65136

Last change on this file since 65136 was 65136, checked in by vboxsync, 8 years ago
VMM/HMVMXR0: Try using the APIC allocated page as the virtual-APIC page.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 583.9 KB

Line
1	/* $Id: HMVMXR0.cpp 65136 2017-01-05 07:06:01Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2016 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#ifdef DEBUG_ramshankar
44	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49	# define HMVMX_ALWAYS_TRAP_PF
50	# define HMVMX_ALWAYS_SWAP_FPU_STATE
51	# define HMVMX_ALWAYS_FLUSH_TLB
52	# define HMVMX_ALWAYS_SWAP_EFER
53	#endif
54
55
56	/*********************************************************************************************************************************
57	* Defined Constants And Macros *
58	*********************************************************************************************************************************/
59	/** Use the function table. */
60	#define HMVMX_USE_FUNCTION_TABLE
61
62	/** Determine which tagged-TLB flush handler to use. */
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68	/** @name Updated-guest-state flags.
69	* @{ */
70	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91	\| HMVMX_UPDATED_GUEST_RSP \
92	\| HMVMX_UPDATED_GUEST_RFLAGS \
93	\| HMVMX_UPDATED_GUEST_CR0 \
94	\| HMVMX_UPDATED_GUEST_CR3 \
95	\| HMVMX_UPDATED_GUEST_CR4 \
96	\| HMVMX_UPDATED_GUEST_GDTR \
97	\| HMVMX_UPDATED_GUEST_IDTR \
98	\| HMVMX_UPDATED_GUEST_LDTR \
99	\| HMVMX_UPDATED_GUEST_TR \
100	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101	\| HMVMX_UPDATED_GUEST_DEBUG \
102	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
107	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108	\| HMVMX_UPDATED_GUEST_INTR_STATE \
109	\| HMVMX_UPDATED_GUEST_APIC_STATE)
110	/** @} */
111
112	/** @name
113	* Flags to skip redundant reads of some common VMCS fields that are not part of
114	* the guest-CPU state but are in the transient structure.
115	*/
116	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123	/** @} */
124
125	/** @name
126	* States of the VMCS.
127	*
128	* This does not reflect all possible VMCS states but currently only those
129	* needed for maintaining the VMCS consistently even when thread-context hooks
130	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
131	*/
132	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135	/** @} */
136
137	/**
138	* Exception bitmap mask for real-mode guests (real-on-v86).
139	*
140	* We need to intercept all exceptions manually except:
141	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143	* due to bugs in Intel CPUs.
144	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
145	* support.
146	*/
147	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
148	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
149	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
150	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
151	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
153	\| RT_BIT(X86_XCPT_XF))
154
155	/**
156	* Exception bitmap mask for all contributory exceptions.
157	*
158	* Page fault is deliberately excluded here as it's conditional as to whether
159	* it's contributory or benign. Page faults are handled separately.
160	*/
161	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
162	\| RT_BIT(X86_XCPT_DE))
163
164	/** Maximum VM-instruction error number. */
165	#define HMVMX_INSTR_ERROR_MAX 28
166
167	/** Profiling macro. */
168	#ifdef HM_PROFILE_EXIT_DISPATCH
169	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171	#else
172	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174	#endif
175
176	/** Assert that preemption is disabled or covered by thread-context hooks. */
177	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180	/** Assert that we haven't migrated CPUs when thread-context hooks are not
181	* used. */
182	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184	("Illegal migration! Entered on CPU %u Current %u\n", \
185	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187	/** Helper macro for VM-exit handlers called unexpectedly. */
188	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189	do { \
190	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191	return VERR_VMX_UNEXPECTED_EXIT; \
192	} while (0)
193
194
195	/*********************************************************************************************************************************
196	* Structures and Typedefs *
197	*********************************************************************************************************************************/
198	/**
199	* VMX transient state.
200	*
201	* A state structure for holding miscellaneous information across
202	* VMX non-root operation and restored after the transition.
203	*/
204	typedef struct VMXTRANSIENT
205	{
206	/** The host's rflags/eflags. */
207	RTCCUINTREG fEFlags;
208	#if HC_ARCH_BITS == 32
209	uint32_t u32Alignment0;
210	#endif
211	/** The guest's TPR value used for TPR shadowing. */
212	uint8_t u8GuestTpr;
213	/** Alignment. */
214	uint8_t abAlignment0[7];
215
216	/** The basic VM-exit reason. */
217	uint16_t uExitReason;
218	/** Alignment. */
219	uint16_t u16Alignment0;
220	/** The VM-exit interruption error code. */
221	uint32_t uExitIntErrorCode;
222	/** The VM-exit exit code qualification. */
223	uint64_t uExitQualification;
224
225	/** The VM-exit interruption-information field. */
226	uint32_t uExitIntInfo;
227	/** The VM-exit instruction-length field. */
228	uint32_t cbInstr;
229	/** The VM-exit instruction-information field. */
230	union
231	{
232	/** Plain unsigned int representation. */
233	uint32_t u;
234	/** INS and OUTS information. */
235	struct
236	{
237	uint32_t u7Reserved0 : 7;
238	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239	uint32_t u3AddrSize : 3;
240	uint32_t u5Reserved1 : 5;
241	/** The segment register (X86_SREG_XXX). */
242	uint32_t iSegReg : 3;
243	uint32_t uReserved2 : 14;
244	} StrIo;
245	} ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Alignment. */
249	uint8_t abAlignment1[3];
250
251	/** The VM-entry interruption-information field. */
252	uint32_t uEntryIntInfo;
253	/** The VM-entry exception error code field. */
254	uint32_t uEntryXcptErrorCode;
255	/** The VM-entry instruction length field. */
256	uint32_t cbEntryInstr;
257
258	/** IDT-vectoring information field. */
259	uint32_t uIdtVectoringInfo;
260	/** IDT-vectoring error code. */
261	uint32_t uIdtVectoringErrorCode;
262
263	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
264	uint32_t fVmcsFieldsRead;
265
266	/** Whether the guest FPU was active at the time of VM-exit. */
267	bool fWasGuestFPUStateActive;
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting should be setup before VM-entry. */
273	bool fUpdateTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	} VMXTRANSIENT;
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286	/** Pointer to VMX transient state. */
287	typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290	/**
291	* MSR-bitmap read permissions.
292	*/
293	typedef enum VMXMSREXITREAD
294	{
295	/** Reading this MSR causes a VM-exit. */
296	VMXMSREXIT_INTERCEPT_READ = 0xb,
297	/** Reading this MSR does not cause a VM-exit. */
298	VMXMSREXIT_PASSTHRU_READ
299	} VMXMSREXITREAD;
300	/** Pointer to MSR-bitmap read permissions. */
301	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303	/**
304	* MSR-bitmap write permissions.
305	*/
306	typedef enum VMXMSREXITWRITE
307	{
308	/** Writing to this MSR causes a VM-exit. */
309	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310	/** Writing to this MSR does not cause a VM-exit. */
311	VMXMSREXIT_PASSTHRU_WRITE
312	} VMXMSREXITWRITE;
313	/** Pointer to MSR-bitmap write permissions. */
314	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317	/**
318	* VMX VM-exit handler.
319	*
320	* @returns Strict VBox status code (i.e. informational status codes too).
321	* @param pVCpu The cross context virtual CPU structure.
322	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
323	* out-of-sync. Make sure to update the required
324	* fields before using them.
325	* @param pVmxTransient Pointer to the VMX-transient structure.
326	*/
327	#ifndef HMVMX_USE_FUNCTION_TABLE
328	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	#else
330	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331	/** Pointer to VM-exit handler. */
332	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333	#endif
334
335	/**
336	* VMX VM-exit handler, non-strict status code.
337	*
338	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339	*
340	* @returns VBox status code, no informational status code returned.
341	* @param pVCpu The cross context virtual CPU structure.
342	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
343	* out-of-sync. Make sure to update the required
344	* fields before using them.
345	* @param pVmxTransient Pointer to the VMX-transient structure.
346	*
347	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348	* use of that status code will be replaced with VINF_EM_SOMETHING
349	* later when switching over to IEM.
350	*/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353	#else
354	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355	#endif
356
357
358	/*********************************************************************************************************************************
359	* Internal Functions *
360	*********************************************************************************************************************************/
361	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366	bool fStepping, uint32_t *puIntState);
367	#if HC_ARCH_BITS == 32
368	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369	#endif
370	#ifndef HMVMX_USE_FUNCTION_TABLE
371	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374	#else
375	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377	#endif
378
379
380	/** @name VM-exit handlers.
381	* @{
382	*/
383	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395	static FNVMXEXITHANDLER hmR0VmxExitHlt;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411	static FNVMXEXITHANDLER hmR0VmxExitMwait;
412	static FNVMXEXITHANDLER hmR0VmxExitMtf;
413	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414	static FNVMXEXITHANDLER hmR0VmxExitPause;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428	/** @} */
429
430	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441	/*********************************************************************************************************************************
442	* Global Variables *
443	*********************************************************************************************************************************/
444	#ifdef HMVMX_USE_FUNCTION_TABLE
445
446	/**
447	* VMX_EXIT dispatch table.
448	*/
449	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450	{
451	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491	/* 40 UNDEFINED */ hmR0VmxExitPause,
492	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
513	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516	};
517	#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519	#ifdef VBOX_STRICT
520	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521	{
522	/* 0 */ "(Not Used)",
523	/* 1 */ "VMCALL executed in VMX root operation.",
524	/* 2 */ "VMCLEAR with invalid physical address.",
525	/* 3 */ "VMCLEAR with VMXON pointer.",
526	/* 4 */ "VMLAUNCH with non-clear VMCS.",
527	/* 5 */ "VMRESUME with non-launched VMCS.",
528	/* 6 */ "VMRESUME after VMXOFF",
529	/* 7 */ "VM-entry with invalid control fields.",
530	/* 8 */ "VM-entry with invalid host state fields.",
531	/* 9 */ "VMPTRLD with invalid physical address.",
532	/* 10 */ "VMPTRLD with VMXON pointer.",
533	/* 11 */ "VMPTRLD with incorrect revision identifier.",
534	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535	/* 13 */ "VMWRITE to read-only VMCS component.",
536	/* 14 */ "(Not Used)",
537	/* 15 */ "VMXON executed in VMX root operation.",
538	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539	/* 17 */ "VM-entry with non-launched executing VMCS.",
540	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541	/* 19 */ "VMCALL with non-clear VMCS.",
542	/* 20 */ "VMCALL with invalid VM-exit control fields.",
543	/* 21 */ "(Not Used)",
544	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546	/* 24 */ "VMCALL with invalid SMM-monitor features.",
547	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548	/* 26 */ "VM-entry with events blocked by MOV SS.",
549	/* 27 */ "(Not Used)",
550	/* 28 */ "Invalid operand to INVEPT/INVVPID."
551	};
552	#endif /* VBOX_STRICT */
553
554
555
556	/**
557	* Updates the VM's last error record.
558	*
559	* If there was a VMX instruction error, reads the error data from the VMCS and
560	* updates VCPU's last error record as well.
561	*
562	* @param pVM The cross context VM structure.
563	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
564	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565	* VERR_VMX_INVALID_VMCS_FIELD.
566	* @param rc The error code.
567	*/
568	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569	{
570	AssertPtr(pVM);
571	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
573	{
574	AssertPtrReturnVoid(pVCpu);
575	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576	}
577	pVM->hm.s.lLastError = rc;
578	}
579
580
581	/**
582	* Reads the VM-entry interruption-information field from the VMCS into the VMX
583	* transient structure.
584	*
585	* @returns VBox status code.
586	* @param pVmxTransient Pointer to the VMX transient structure.
587	*
588	* @remarks No-long-jump zone!!!
589	*/
590	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591	{
592	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593	AssertRCReturn(rc, rc);
594	return VINF_SUCCESS;
595	}
596
597
598	#ifdef VBOX_STRICT
599	/**
600	* Reads the VM-entry exception error code field from the VMCS into
601	* the VMX transient structure.
602	*
603	* @returns VBox status code.
604	* @param pVmxTransient Pointer to the VMX transient structure.
605	*
606	* @remarks No-long-jump zone!!!
607	*/
608	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609	{
610	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611	AssertRCReturn(rc, rc);
612	return VINF_SUCCESS;
613	}
614	#endif /* VBOX_STRICT */
615
616
617	#ifdef VBOX_STRICT
618	/**
619	* Reads the VM-entry exception error code field from the VMCS into
620	* the VMX transient structure.
621	*
622	* @returns VBox status code.
623	* @param pVmxTransient Pointer to the VMX transient structure.
624	*
625	* @remarks No-long-jump zone!!!
626	*/
627	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628	{
629	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630	AssertRCReturn(rc, rc);
631	return VINF_SUCCESS;
632	}
633	#endif /* VBOX_STRICT */
634
635
636	/**
637	* Reads the VM-exit interruption-information field from the VMCS into the VMX
638	* transient structure.
639	*
640	* @returns VBox status code.
641	* @param pVmxTransient Pointer to the VMX transient structure.
642	*/
643	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644	{
645	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646	{
647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648	AssertRCReturn(rc, rc);
649	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650	}
651	return VINF_SUCCESS;
652	}
653
654
655	/**
656	* Reads the VM-exit interruption error code from the VMCS into the VMX
657	* transient structure.
658	*
659	* @returns VBox status code.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665	{
666	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the VM-exit instruction length field from the VMCS into the VMX
676	* transient structure.
677	*
678	* @returns VBox status code.
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684	{
685	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the VM-exit instruction-information field from the VMCS into
695	* the VMX transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*/
700	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701	{
702	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703	{
704	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705	AssertRCReturn(rc, rc);
706	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707	}
708	return VINF_SUCCESS;
709	}
710
711
712	/**
713	* Reads the exit code qualification from the VMCS into the VMX transient
714	* structure.
715	*
716	* @returns VBox status code.
717	* @param pVCpu The cross context virtual CPU structure of the
718	* calling EMT. (Required for the VMCS cache case.)
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724	{
725	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Reads the IDT-vectoring information field from the VMCS into the VMX
735	* transient structure.
736	*
737	* @returns VBox status code.
738	* @param pVmxTransient Pointer to the VMX transient structure.
739	*
740	* @remarks No-long-jump zone!!!
741	*/
742	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743	{
744	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745	{
746	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747	AssertRCReturn(rc, rc);
748	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749	}
750	return VINF_SUCCESS;
751	}
752
753
754	/**
755	* Reads the IDT-vectoring error code from the VMCS into the VMX
756	* transient structure.
757	*
758	* @returns VBox status code.
759	* @param pVmxTransient Pointer to the VMX transient structure.
760	*/
761	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762	{
763	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764	{
765	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766	AssertRCReturn(rc, rc);
767	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768	}
769	return VINF_SUCCESS;
770	}
771
772
773	/**
774	* Enters VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	* @param pVM The cross context VM structure. Can be
778	* NULL, after a resume.
779	* @param HCPhysCpuPage Physical address of the VMXON region.
780	* @param pvCpuPage Pointer to the VMXON region.
781	*/
782	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783	{
784	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786	Assert(pvCpuPage);
787	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789	if (pVM)
790	{
791	/* Write the VMCS revision dword to the VMXON region. */
792	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793	}
794
795	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796	RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798	/* Enable the VMX bit in CR4 if necessary. */
799	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801	/* Enter VMX root mode. */
802	int rc = VMXEnable(HCPhysCpuPage);
803	if (RT_FAILURE(rc))
804	{
805	if (!(uOldCr4 & X86_CR4_VMXE))
806	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808	if (pVM)
809	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810	}
811
812	/* Restore interrupts. */
813	ASMSetFlags(fEFlags);
814	return rc;
815	}
816
817
818	/**
819	* Exits VMX root mode operation on the current CPU.
820	*
821	* @returns VBox status code.
822	*/
823	static int hmR0VmxLeaveRootMode(void)
824	{
825	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828	RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830	/* If we're for some reason not in VMX root mode, then don't leave it. */
831	RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833	int rc;
834	if (uHostCR4 & X86_CR4_VMXE)
835	{
836	/* Exit VMX root mode and clear the VMX bit in CR4. */
837	VMXDisable();
838	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839	rc = VINF_SUCCESS;
840	}
841	else
842	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844	/* Restore interrupts. */
845	ASMSetFlags(fEFlags);
846	return rc;
847	}
848
849
850	/**
851	* Allocates and maps one physically contiguous page. The allocated page is
852	* zero'd out. (Used by various VT-x structures).
853	*
854	* @returns IPRT status code.
855	* @param pMemObj Pointer to the ring-0 memory object.
856	* @param ppVirt Where to store the virtual address of the
857	* allocation.
858	* @param pHCPhys Where to store the physical address of the
859	* allocation.
860	*/
861	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862	{
863	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868	if (RT_FAILURE(rc))
869	return rc;
870	ppVirt = RTR0MemObjAddress(pMemObj);
871	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
872	ASMMemZero32(*ppVirt, PAGE_SIZE);
873	return VINF_SUCCESS;
874	}
875
876
877	/**
878	* Frees and unmaps an allocated physical page.
879	*
880	* @param pMemObj Pointer to the ring-0 memory object.
881	* @param ppVirt Where to re-initialize the virtual address of
882	* allocation as 0.
883	* @param pHCPhys Where to re-initialize the physical address of the
884	* allocation as 0.
885	*/
886	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887	{
888	AssertPtr(pMemObj);
889	AssertPtr(ppVirt);
890	AssertPtr(pHCPhys);
891	if (*pMemObj != NIL_RTR0MEMOBJ)
892	{
893	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
894	AssertRC(rc);
895	*pMemObj = NIL_RTR0MEMOBJ;
896	*ppVirt = 0;
897	*pHCPhys = 0;
898	}
899	}
900
901
902	/**
903	* Worker function to free VT-x related structures.
904	*
905	* @returns IPRT status code.
906	* @param pVM The cross context VM structure.
907	*/
908	static void hmR0VmxStructsFree(PVM pVM)
909	{
910	for (VMCPUID i = 0; i < pVM->cCpus; i++)
911	{
912	PVMCPU pVCpu = &pVM->aCpus[i];
913	AssertPtr(pVCpu);
914
915	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922	}
923
924	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927	#endif
928	}
929
930
931	/**
932	* Worker function to allocate VT-x related VM structures.
933	*
934	* @returns IPRT status code.
935	* @param pVM The cross context VM structure.
936	*/
937	static int hmR0VmxStructsAlloc(PVM pVM)
938	{
939	/*
940	* Initialize members up-front so we can cleanup properly on allocation failure.
941	*/
942	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945	pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954	#endif
955	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958	for (VMCPUID i = 0; i < pVM->cCpus; i++)
959	{
960	PVMCPU pVCpu = &pVM->aCpus[i];
961	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
965	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
966	}
967	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
968	#undef VMXLOCAL_INIT_VM_MEMOBJ
969
970	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
971	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
972	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
973	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
974
975	/*
976	* Allocate all the VT-x structures.
977	*/
978	int rc = VINF_SUCCESS;
979	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
980	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
981	if (RT_FAILURE(rc))
982	goto cleanup;
983	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
984	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
985	#endif
986
987	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
988	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
989	{
990	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
991	&pVM->hm.s.vmx.HCPhysApicAccess);
992	if (RT_FAILURE(rc))
993	goto cleanup;
994	}
995
996	/*
997	* Initialize per-VCPU VT-x structures.
998	*/
999	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1000	{
1001	PVMCPU pVCpu = &pVM->aCpus[i];
1002	AssertPtr(pVCpu);
1003
1004	/* Allocate the VM control structure (VMCS). */
1005	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1006	if (RT_FAILURE(rc))
1007	goto cleanup;
1008
1009	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
1010	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1011	{
1012	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013	NULL /* pR3Ptr /, NULL / pRCPtr */);
1014	if (RT_FAILURE(rc))
1015	goto cleanup;
1016	}
1017
1018	/*
1019	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020	* transparent accesses of specific MSRs.
1021	*
1022	* If the condition for enabling MSR bitmaps changes here, don't forget to
1023	* update HMAreMsrBitmapsAvailable().
1024	*/
1025	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026	{
1027	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029	if (RT_FAILURE(rc))
1030	goto cleanup;
1031	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032	}
1033
1034	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036	if (RT_FAILURE(rc))
1037	goto cleanup;
1038
1039	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1040	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041	if (RT_FAILURE(rc))
1042	goto cleanup;
1043	}
1044
1045	return VINF_SUCCESS;
1046
1047	cleanup:
1048	hmR0VmxStructsFree(pVM);
1049	return rc;
1050	}
1051
1052
1053	/**
1054	* Does global VT-x initialization (called during module initialization).
1055	*
1056	* @returns VBox status code.
1057	*/
1058	VMMR0DECL(int) VMXR0GlobalInit(void)
1059	{
1060	#ifdef HMVMX_USE_FUNCTION_TABLE
1061	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062	# ifdef VBOX_STRICT
1063	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064	Assert(g_apfnVMExitHandlers[i]);
1065	# endif
1066	#endif
1067	return VINF_SUCCESS;
1068	}
1069
1070
1071	/**
1072	* Does global VT-x termination (called during module termination).
1073	*/
1074	VMMR0DECL(void) VMXR0GlobalTerm()
1075	{
1076	/* Nothing to do currently. */
1077	}
1078
1079
1080	/**
1081	* Sets up and activates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pVM The cross context VM structure. Can be
1086	* NULL after a host resume operation.
1087	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088	* fEnabledByHost is @c true).
1089	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090	* @a fEnabledByHost is @c true).
1091	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092	* enable VT-x on the host.
1093	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1094	*/
1095	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096	void *pvMsrs)
1097	{
1098	Assert(pCpu);
1099	Assert(pvMsrs);
1100	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102	/* Enable VT-x if it's not already enabled by the host. */
1103	if (!fEnabledByHost)
1104	{
1105	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106	if (RT_FAILURE(rc))
1107	return rc;
1108	}
1109
1110	/*
1111	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1112	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1113	*/
1114	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1115	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1116	{
1117	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1118	pCpu->fFlushAsidBeforeUse = false;
1119	}
1120	else
1121	pCpu->fFlushAsidBeforeUse = true;
1122
1123	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1124	++pCpu->cTlbFlushes;
1125
1126	return VINF_SUCCESS;
1127	}
1128
1129
1130	/**
1131	* Deactivates VT-x on the current CPU.
1132	*
1133	* @returns VBox status code.
1134	* @param pCpu Pointer to the global CPU info struct.
1135	* @param pvCpuPage Pointer to the VMXON region.
1136	* @param HCPhysCpuPage Physical address of the VMXON region.
1137	*
1138	* @remarks This function should never be called when SUPR0EnableVTx() or
1139	* similar was used to enable VT-x on the host.
1140	*/
1141	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1142	{
1143	NOREF(pCpu);
1144	NOREF(pvCpuPage);
1145	NOREF(HCPhysCpuPage);
1146
1147	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1148	return hmR0VmxLeaveRootMode();
1149	}
1150
1151
1152	/**
1153	* Sets the permission bits for the specified MSR in the MSR bitmap.
1154	*
1155	* @param pVCpu The cross context virtual CPU structure.
1156	* @param uMsr The MSR value.
1157	* @param enmRead Whether reading this MSR causes a VM-exit.
1158	* @param enmWrite Whether writing this MSR causes a VM-exit.
1159	*/
1160	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1161	{
1162	int32_t iBit;
1163	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1164
1165	/*
1166	* Layout:
1167	* 0x000 - 0x3ff - Low MSR read bits
1168	* 0x400 - 0x7ff - High MSR read bits
1169	* 0x800 - 0xbff - Low MSR write bits
1170	* 0xc00 - 0xfff - High MSR write bits
1171	*/
1172	if (uMsr <= 0x00001FFF)
1173	iBit = uMsr;
1174	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1175	{
1176	iBit = uMsr - UINT32_C(0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1181
1182	Assert(iBit <= 0x1fff);
1183	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1184	ASMBitSet(pbMsrBitmap, iBit);
1185	else
1186	ASMBitClear(pbMsrBitmap, iBit);
1187
1188	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1189	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1190	else
1191	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1192	}
1193
1194
1195	#ifdef VBOX_STRICT
1196	/**
1197	* Gets the permission bits for the specified MSR in the MSR bitmap.
1198	*
1199	* @returns VBox status code.
1200	* @retval VINF_SUCCESS if the specified MSR is found.
1201	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1202	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1203	*
1204	* @param pVCpu The cross context virtual CPU structure.
1205	* @param uMsr The MSR.
1206	* @param penmRead Where to store the read permissions.
1207	* @param penmWrite Where to store the write permissions.
1208	*/
1209	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1210	{
1211	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1212	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1213	int32_t iBit;
1214	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1215
1216	/* See hmR0VmxSetMsrPermission() for the layout. */
1217	if (uMsr <= 0x00001FFF)
1218	iBit = uMsr;
1219	else if ( uMsr >= 0xC0000000
1220	&& uMsr <= 0xC0001FFF)
1221	{
1222	iBit = (uMsr - 0xC0000000);
1223	pbMsrBitmap += 0x400;
1224	}
1225	else
1226	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1227
1228	Assert(iBit <= 0x1fff);
1229	if (ASMBitTest(pbMsrBitmap, iBit))
1230	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1231	else
1232	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1233
1234	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1235	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1236	else
1237	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1238	return VINF_SUCCESS;
1239	}
1240	#endif /* VBOX_STRICT */
1241
1242
1243	/**
1244	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1245	* area.
1246	*
1247	* @returns VBox status code.
1248	* @param pVCpu The cross context virtual CPU structure.
1249	* @param cMsrs The number of MSRs.
1250	*/
1251	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1252	{
1253	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1254	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1255	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1256	{
1257	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1258	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1259	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1260	}
1261
1262	/* Update number of guest MSRs to load/store across the world-switch. */
1263	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1264	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1265
1266	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1267	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1268	AssertRCReturn(rc, rc);
1269
1270	/* Update the VCPU's copy of the MSR count. */
1271	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1272
1273	return VINF_SUCCESS;
1274	}
1275
1276
1277	/**
1278	* Adds a new (or updates the value of an existing) guest/host MSR
1279	* pair to be swapped during the world-switch as part of the
1280	* auto-load/store MSR area in the VMCS.
1281	*
1282	* @returns VBox status code.
1283	* @param pVCpu The cross context virtual CPU structure.
1284	* @param uMsr The MSR.
1285	* @param uGuestMsrValue Value of the guest MSR.
1286	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1287	* necessary.
1288	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1289	* its value was updated. Optional, can be NULL.
1290	*/
1291	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1292	bool *pfAddedAndUpdated)
1293	{
1294	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1295	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1296	uint32_t i;
1297	for (i = 0; i < cMsrs; i++)
1298	{
1299	if (pGuestMsr->u32Msr == uMsr)
1300	break;
1301	pGuestMsr++;
1302	}
1303
1304	bool fAdded = false;
1305	if (i == cMsrs)
1306	{
1307	++cMsrs;
1308	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1309	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1310
1311	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1312	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1313	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1314
1315	fAdded = true;
1316	}
1317
1318	/* Update the MSR values in the auto-load/store MSR area. */
1319	pGuestMsr->u32Msr = uMsr;
1320	pGuestMsr->u64Value = uGuestMsrValue;
1321
1322	/* Create/update the MSR slot in the host MSR area. */
1323	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1324	pHostMsr += i;
1325	pHostMsr->u32Msr = uMsr;
1326
1327	/*
1328	* Update the host MSR only when requested by the caller AND when we're
1329	* adding it to the auto-load/store area. Otherwise, it would have been
1330	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1331	*/
1332	bool fUpdatedMsrValue = false;
1333	if ( fAdded
1334	&& fUpdateHostMsr)
1335	{
1336	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1337	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1338	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1339	fUpdatedMsrValue = true;
1340	}
1341
1342	if (pfAddedAndUpdated)
1343	*pfAddedAndUpdated = fUpdatedMsrValue;
1344	return VINF_SUCCESS;
1345	}
1346
1347
1348	/**
1349	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1350	* auto-load/store MSR area in the VMCS.
1351	*
1352	* @returns VBox status code.
1353	* @param pVCpu The cross context virtual CPU structure.
1354	* @param uMsr The MSR.
1355	*/
1356	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1357	{
1358	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1359	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1360	for (uint32_t i = 0; i < cMsrs; i++)
1361	{
1362	/* Find the MSR. */
1363	if (pGuestMsr->u32Msr == uMsr)
1364	{
1365	/* If it's the last MSR, simply reduce the count. */
1366	if (i == cMsrs - 1)
1367	{
1368	--cMsrs;
1369	break;
1370	}
1371
1372	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1373	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1374	pLastGuestMsr += cMsrs - 1;
1375	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1376	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1377
1378	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380	pLastHostMsr += cMsrs - 1;
1381	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1382	pHostMsr->u64Value = pLastHostMsr->u64Value;
1383	--cMsrs;
1384	break;
1385	}
1386	pGuestMsr++;
1387	}
1388
1389	/* Update the VMCS if the count changed (meaning the MSR was found). */
1390	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1391	{
1392	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1393	AssertRCReturn(rc, rc);
1394
1395	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1396	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1397	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1398
1399	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1400	return VINF_SUCCESS;
1401	}
1402
1403	return VERR_NOT_FOUND;
1404	}
1405
1406
1407	/**
1408	* Checks if the specified guest MSR is part of the auto-load/store area in
1409	* the VMCS.
1410	*
1411	* @returns true if found, false otherwise.
1412	* @param pVCpu The cross context virtual CPU structure.
1413	* @param uMsr The MSR to find.
1414	*/
1415	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1416	{
1417	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1418	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1419
1420	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1421	{
1422	if (pGuestMsr->u32Msr == uMsr)
1423	return true;
1424	}
1425	return false;
1426	}
1427
1428
1429	/**
1430	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1431	*
1432	* @param pVCpu The cross context virtual CPU structure.
1433	*
1434	* @remarks No-long-jump zone!!!
1435	*/
1436	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1437	{
1438	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1439	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1440	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1441	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1442
1443	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1444	{
1445	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1446
1447	/*
1448	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1449	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1450	*/
1451	if (pHostMsr->u32Msr == MSR_K6_EFER)
1452	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1453	else
1454	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1455	}
1456
1457	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1458	}
1459
1460
1461	/**
1462	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1463	* perform lazy restoration of the host MSRs while leaving VT-x.
1464	*
1465	* @param pVCpu The cross context virtual CPU structure.
1466	*
1467	* @remarks No-long-jump zone!!!
1468	*/
1469	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1470	{
1471	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1472
1473	/*
1474	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1475	*/
1476	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477	{
1478	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1479	#if HC_ARCH_BITS == 64
1480	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1481	{
1482	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1483	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1484	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1485	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1486	}
1487	#endif
1488	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1489	}
1490	}
1491
1492
1493	/**
1494	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1495	* lazily while leaving VT-x.
1496	*
1497	* @returns true if it does, false otherwise.
1498	* @param pVCpu The cross context virtual CPU structure.
1499	* @param uMsr The MSR to check.
1500	*/
1501	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1502	{
1503	NOREF(pVCpu);
1504	#if HC_ARCH_BITS == 64
1505	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1506	{
1507	switch (uMsr)
1508	{
1509	case MSR_K8_LSTAR:
1510	case MSR_K6_STAR:
1511	case MSR_K8_SF_MASK:
1512	case MSR_K8_KERNEL_GS_BASE:
1513	return true;
1514	}
1515	}
1516	#else
1517	RT_NOREF(pVCpu, uMsr);
1518	#endif
1519	return false;
1520	}
1521
1522
1523	/**
1524	* Saves a set of guest MSRs back into the guest-CPU context.
1525	*
1526	* @param pVCpu The cross context virtual CPU structure.
1527	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1528	* out-of-sync. Make sure to update the required fields
1529	* before using them.
1530	*
1531	* @remarks No-long-jump zone!!!
1532	*/
1533	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1534	{
1535	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1536	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1537
1538	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1539	{
1540	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1541	#if HC_ARCH_BITS == 64
1542	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1543	{
1544	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1545	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1546	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1547	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1548	}
1549	#else
1550	NOREF(pMixedCtx);
1551	#endif
1552	}
1553	}
1554
1555
1556	/**
1557	* Loads a set of guests MSRs to allow read/passthru to the guest.
1558	*
1559	* The name of this function is slightly confusing. This function does NOT
1560	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1561	* common prefix for functions dealing with "lazy restoration" of the shared
1562	* MSRs.
1563	*
1564	* @param pVCpu The cross context virtual CPU structure.
1565	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1566	* out-of-sync. Make sure to update the required fields
1567	* before using them.
1568	*
1569	* @remarks No-long-jump zone!!!
1570	*/
1571	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1572	{
1573	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1574	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1575
1576	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1577	do { \
1578	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1579	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1580	else \
1581	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1582	} while (0)
1583
1584	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1585	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1586	{
1587	#if HC_ARCH_BITS == 64
1588	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1589	{
1590	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1591	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1592	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1593	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1594	}
1595	#else
1596	RT_NOREF(pMixedCtx);
1597	#endif
1598	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1599	}
1600
1601	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1602	}
1603
1604
1605	/**
1606	* Performs lazy restoration of the set of host MSRs if they were previously
1607	* loaded with guest MSR values.
1608	*
1609	* @param pVCpu The cross context virtual CPU structure.
1610	*
1611	* @remarks No-long-jump zone!!!
1612	* @remarks The guest MSRs should have been saved back into the guest-CPU
1613	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1614	*/
1615	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1616	{
1617	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1618	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1619
1620	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1621	{
1622	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1623	#if HC_ARCH_BITS == 64
1624	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1625	{
1626	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1627	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1628	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1629	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1630	}
1631	#endif
1632	}
1633	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1634	}
1635
1636
1637	/**
1638	* Verifies that our cached values of the VMCS controls are all
1639	* consistent with what's actually present in the VMCS.
1640	*
1641	* @returns VBox status code.
1642	* @param pVCpu The cross context virtual CPU structure.
1643	*/
1644	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1645	{
1646	uint32_t u32Val;
1647	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1648	AssertRCReturn(rc, rc);
1649	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1650	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1651
1652	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1653	AssertRCReturn(rc, rc);
1654	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1655	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1656
1657	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1658	AssertRCReturn(rc, rc);
1659	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1660	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1661
1662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1663	AssertRCReturn(rc, rc);
1664	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1665	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1666
1667	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1668	{
1669	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1670	AssertRCReturn(rc, rc);
1671	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1672	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1673	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1674	}
1675
1676	return VINF_SUCCESS;
1677	}
1678
1679
1680	#ifdef VBOX_STRICT
1681	/**
1682	* Verifies that our cached host EFER value has not changed
1683	* since we cached it.
1684	*
1685	* @param pVCpu The cross context virtual CPU structure.
1686	*/
1687	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1688	{
1689	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1690
1691	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1692	{
1693	uint64_t u64Val;
1694	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1695	AssertRC(rc);
1696
1697	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1698	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1699	}
1700	}
1701
1702
1703	/**
1704	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1705	* VMCS are correct.
1706	*
1707	* @param pVCpu The cross context virtual CPU structure.
1708	*/
1709	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1710	{
1711	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1712
1713	/* Verify MSR counts in the VMCS are what we think it should be. */
1714	uint32_t cMsrs;
1715	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1716	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1717
1718	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1719	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1720
1721	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1722	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1723
1724	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1725	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1726	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1727	{
1728	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1729	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1730	pGuestMsr->u32Msr, cMsrs));
1731
1732	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1733	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1734	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1735
1736	/* Verify that the permissions are as expected in the MSR bitmap. */
1737	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1738	{
1739	VMXMSREXITREAD enmRead;
1740	VMXMSREXITWRITE enmWrite;
1741	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1742	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1743	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1744	{
1745	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1746	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1747	}
1748	else
1749	{
1750	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1751	pGuestMsr->u32Msr, cMsrs));
1752	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1753	pGuestMsr->u32Msr, cMsrs));
1754	}
1755	}
1756	}
1757	}
1758	#endif /* VBOX_STRICT */
1759
1760
1761	/**
1762	* Flushes the TLB using EPT.
1763	*
1764	* @returns VBox status code.
1765	* @param pVCpu The cross context virtual CPU structure of the calling
1766	* EMT. Can be NULL depending on @a enmFlush.
1767	* @param enmFlush Type of flush.
1768	*
1769	* @remarks Caller is responsible for making sure this function is called only
1770	* when NestedPaging is supported and providing @a enmFlush that is
1771	* supported by the CPU.
1772	* @remarks Can be called with interrupts disabled.
1773	*/
1774	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1775	{
1776	uint64_t au64Descriptor[2];
1777	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1778	au64Descriptor[0] = 0;
1779	else
1780	{
1781	Assert(pVCpu);
1782	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1783	}
1784	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1785
1786	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1787	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1788	rc));
1789	if ( RT_SUCCESS(rc)
1790	&& pVCpu)
1791	{
1792	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1793	}
1794	}
1795
1796
1797	/**
1798	* Flushes the TLB using VPID.
1799	*
1800	* @returns VBox status code.
1801	* @param pVM The cross context VM structure.
1802	* @param pVCpu The cross context virtual CPU structure of the calling
1803	* EMT. Can be NULL depending on @a enmFlush.
1804	* @param enmFlush Type of flush.
1805	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1806	* on @a enmFlush).
1807	*
1808	* @remarks Can be called with interrupts disabled.
1809	*/
1810	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1811	{
1812	NOREF(pVM);
1813	AssertPtr(pVM);
1814	Assert(pVM->hm.s.vmx.fVpid);
1815
1816	uint64_t au64Descriptor[2];
1817	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1818	{
1819	au64Descriptor[0] = 0;
1820	au64Descriptor[1] = 0;
1821	}
1822	else
1823	{
1824	AssertPtr(pVCpu);
1825	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1827	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1828	au64Descriptor[1] = GCPtr;
1829	}
1830
1831	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1832	AssertMsg(rc == VINF_SUCCESS,
1833	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1834	if ( RT_SUCCESS(rc)
1835	&& pVCpu)
1836	{
1837	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1838	}
1839	}
1840
1841
1842	/**
1843	* Invalidates a guest page by guest virtual address. Only relevant for
1844	* EPT/VPID, otherwise there is nothing really to invalidate.
1845	*
1846	* @returns VBox status code.
1847	* @param pVM The cross context VM structure.
1848	* @param pVCpu The cross context virtual CPU structure.
1849	* @param GCVirt Guest virtual address of the page to invalidate.
1850	*/
1851	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1852	{
1853	AssertPtr(pVM);
1854	AssertPtr(pVCpu);
1855	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1856
1857	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1858	if (!fFlushPending)
1859	{
1860	/*
1861	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1862	* See @bugref{6043} and @bugref{6177}.
1863	*
1864	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1865	* function maybe called in a loop with individual addresses.
1866	*/
1867	if (pVM->hm.s.vmx.fVpid)
1868	{
1869	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1870	{
1871	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1872	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1873	}
1874	else
1875	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1876	}
1877	else if (pVM->hm.s.fNestedPaging)
1878	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1879	}
1880
1881	return VINF_SUCCESS;
1882	}
1883
1884
1885	/**
1886	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1887	* otherwise there is nothing really to invalidate.
1888	*
1889	* @returns VBox status code.
1890	* @param pVM The cross context VM structure.
1891	* @param pVCpu The cross context virtual CPU structure.
1892	* @param GCPhys Guest physical address of the page to invalidate.
1893	*/
1894	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1895	{
1896	NOREF(pVM); NOREF(GCPhys);
1897	LogFlowFunc(("%RGp\n", GCPhys));
1898
1899	/*
1900	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1901	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1902	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1903	*/
1904	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1905	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1912	* case where neither EPT nor VPID is supported by the CPU.
1913	*
1914	* @param pVM The cross context VM structure.
1915	* @param pVCpu The cross context virtual CPU structure.
1916	* @param pCpu Pointer to the global HM struct.
1917	*
1918	* @remarks Called with interrupts disabled.
1919	*/
1920	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1921	{
1922	AssertPtr(pVCpu);
1923	AssertPtr(pCpu);
1924	NOREF(pVM);
1925
1926	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1927
1928	Assert(pCpu->idCpu != NIL_RTCPUID);
1929	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1930	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1931	pVCpu->hm.s.fForceTLBFlush = false;
1932	return;
1933	}
1934
1935
1936	/**
1937	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1938	*
1939	* @param pVM The cross context VM structure.
1940	* @param pVCpu The cross context virtual CPU structure.
1941	* @param pCpu Pointer to the global HM CPU struct.
1942	* @remarks All references to "ASID" in this function pertains to "VPID" in
1943	* Intel's nomenclature. The reason is, to avoid confusion in compare
1944	* statements since the host-CPU copies are named "ASID".
1945	*
1946	* @remarks Called with interrupts disabled.
1947	*/
1948	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1949	{
1950	#ifdef VBOX_WITH_STATISTICS
1951	bool fTlbFlushed = false;
1952	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1953	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1954	if (!fTlbFlushed) \
1955	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1956	} while (0)
1957	#else
1958	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1959	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1960	#endif
1961
1962	AssertPtr(pVM);
1963	AssertPtr(pCpu);
1964	AssertPtr(pVCpu);
1965	Assert(pCpu->idCpu != NIL_RTCPUID);
1966
1967	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1968	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1969	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1970
1971	/*
1972	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1973	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1974	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1975	*/
1976	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1977	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1978	{
1979	++pCpu->uCurrentAsid;
1980	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1981	{
1982	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1983	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1984	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1985	}
1986
1987	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1988	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1989	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1990
1991	/*
1992	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1993	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1994	*/
1995	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1996	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1997	HMVMX_SET_TAGGED_TLB_FLUSHED();
1998	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1999	}
2000
2001	/* Check for explicit TLB flushes. */
2002	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2003	{
2004	/*
2005	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2006	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2007	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2008	* but not guest-physical mappings.
2009	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2010	*/
2011	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2012	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2013	HMVMX_SET_TAGGED_TLB_FLUSHED();
2014	}
2015
2016	pVCpu->hm.s.fForceTLBFlush = false;
2017	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2018
2019	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2020	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2021	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2022	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2023	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2024	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2025	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2026	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2027	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2028
2029	/* Update VMCS with the VPID. */
2030	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2031	AssertRC(rc);
2032
2033	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2034	}
2035
2036
2037	/**
2038	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2039	*
2040	* @returns VBox status code.
2041	* @param pVM The cross context VM structure.
2042	* @param pVCpu The cross context virtual CPU structure.
2043	* @param pCpu Pointer to the global HM CPU struct.
2044	*
2045	* @remarks Called with interrupts disabled.
2046	*/
2047	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2048	{
2049	AssertPtr(pVM);
2050	AssertPtr(pVCpu);
2051	AssertPtr(pCpu);
2052	Assert(pCpu->idCpu != NIL_RTCPUID);
2053	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2054	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2055
2056	/*
2057	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2058	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2059	*/
2060	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2061	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2062	{
2063	pVCpu->hm.s.fForceTLBFlush = true;
2064	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2065	}
2066
2067	/* Check for explicit TLB flushes. */
2068	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2069	{
2070	pVCpu->hm.s.fForceTLBFlush = true;
2071	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2072	}
2073
2074	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2075	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2076
2077	if (pVCpu->hm.s.fForceTLBFlush)
2078	{
2079	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2080	pVCpu->hm.s.fForceTLBFlush = false;
2081	}
2082	}
2083
2084
2085	/**
2086	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2087	*
2088	* @returns VBox status code.
2089	* @param pVM The cross context VM structure.
2090	* @param pVCpu The cross context virtual CPU structure.
2091	* @param pCpu Pointer to the global HM CPU struct.
2092	*
2093	* @remarks Called with interrupts disabled.
2094	*/
2095	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2096	{
2097	AssertPtr(pVM);
2098	AssertPtr(pVCpu);
2099	AssertPtr(pCpu);
2100	Assert(pCpu->idCpu != NIL_RTCPUID);
2101	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2102	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2103
2104	/*
2105	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2106	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2107	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2108	*/
2109	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2110	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2111	{
2112	pVCpu->hm.s.fForceTLBFlush = true;
2113	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2114	}
2115
2116	/* Check for explicit TLB flushes. */
2117	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2118	{
2119	/*
2120	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2121	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2122	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2123	*/
2124	pVCpu->hm.s.fForceTLBFlush = true;
2125	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2126	}
2127
2128	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2129	if (pVCpu->hm.s.fForceTLBFlush)
2130	{
2131	++pCpu->uCurrentAsid;
2132	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2133	{
2134	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2135	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2136	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2137	}
2138
2139	pVCpu->hm.s.fForceTLBFlush = false;
2140	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2141	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2142	if (pCpu->fFlushAsidBeforeUse)
2143	{
2144	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2145	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2146	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2147	{
2148	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2149	pCpu->fFlushAsidBeforeUse = false;
2150	}
2151	else
2152	{
2153	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2154	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2155	}
2156	}
2157	}
2158
2159	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2160	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2161	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2162	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2163	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2164	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2165	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2166
2167	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2168	AssertRC(rc);
2169	}
2170
2171
2172	/**
2173	* Flushes the guest TLB entry based on CPU capabilities.
2174	*
2175	* @param pVCpu The cross context virtual CPU structure.
2176	* @param pCpu Pointer to the global HM CPU struct.
2177	*/
2178	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2179	{
2180	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2181	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2182	#endif
2183	PVM pVM = pVCpu->CTX_SUFF(pVM);
2184	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2185	{
2186	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2187	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2188	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2189	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2190	default:
2191	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2192	break;
2193	}
2194
2195	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2196	}
2197
2198
2199	/**
2200	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2201	* TLB entries from the host TLB before VM-entry.
2202	*
2203	* @returns VBox status code.
2204	* @param pVM The cross context VM structure.
2205	*/
2206	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2207	{
2208	/*
2209	* Determine optimal flush type for Nested Paging.
2210	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2211	* guest execution (see hmR3InitFinalizeR0()).
2212	*/
2213	if (pVM->hm.s.fNestedPaging)
2214	{
2215	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2216	{
2217	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2218	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2219	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2220	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2221	else
2222	{
2223	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2224	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2225	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2226	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2227	}
2228
2229	/* Make sure the write-back cacheable memory type for EPT is supported. */
2230	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2231	{
2232	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2233	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2234	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2235	}
2236
2237	/* EPT requires a page-walk length of 4. */
2238	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2239	{
2240	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2241	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2242	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2243	}
2244	}
2245	else
2246	{
2247	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2248	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2249	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2250	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2251	}
2252	}
2253
2254	/*
2255	* Determine optimal flush type for VPID.
2256	*/
2257	if (pVM->hm.s.vmx.fVpid)
2258	{
2259	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2260	{
2261	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2262	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2263	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2264	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2265	else
2266	{
2267	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2268	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2269	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2270	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2271	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2272	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2273	pVM->hm.s.vmx.fVpid = false;
2274	}
2275	}
2276	else
2277	{
2278	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2279	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2280	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2281	pVM->hm.s.vmx.fVpid = false;
2282	}
2283	}
2284
2285	/*
2286	* Setup the handler for flushing tagged-TLBs.
2287	*/
2288	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2289	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2290	else if (pVM->hm.s.fNestedPaging)
2291	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2292	else if (pVM->hm.s.vmx.fVpid)
2293	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2294	else
2295	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2296	return VINF_SUCCESS;
2297	}
2298
2299
2300	/**
2301	* Sets up pin-based VM-execution controls in the VMCS.
2302	*
2303	* @returns VBox status code.
2304	* @param pVM The cross context VM structure.
2305	* @param pVCpu The cross context virtual CPU structure.
2306	*/
2307	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2308	{
2309	AssertPtr(pVM);
2310	AssertPtr(pVCpu);
2311
2312	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2313	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2314
2315	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2316	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2317
2318	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2319	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2320
2321	/* Enable the VMX preemption timer. */
2322	if (pVM->hm.s.vmx.fUsePreemptTimer)
2323	{
2324	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2325	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2326	}
2327
2328	#if 0
2329	/* Enable posted-interrupt processing. */
2330	if (pVM->hm.s.fPostedIntrs)
2331	{
2332	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2333	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2334	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2335	}
2336	#endif
2337
2338	if ((val & zap) != val)
2339	{
2340	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2341	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2342	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2343	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2344	}
2345
2346	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2347	AssertRCReturn(rc, rc);
2348
2349	pVCpu->hm.s.vmx.u32PinCtls = val;
2350	return rc;
2351	}
2352
2353
2354	/**
2355	* Sets up processor-based VM-execution controls in the VMCS.
2356	*
2357	* @returns VBox status code.
2358	* @param pVM The cross context VM structure.
2359	* @param pVCpu The cross context virtual CPU structure.
2360	*/
2361	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2362	{
2363	AssertPtr(pVM);
2364	AssertPtr(pVCpu);
2365
2366	int rc = VERR_INTERNAL_ERROR_5;
2367	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2368	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2369
2370	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2371	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2372	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2373	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2374	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2375	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2376	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2377
2378	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2379	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2380	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2381	{
2382	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2383	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2384	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2385	}
2386
2387	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2388	if (!pVM->hm.s.fNestedPaging)
2389	{
2390	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2392	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2393	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2394	}
2395
2396	/* Use TPR shadowing if supported by the CPU. */
2397	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2398	{
2399	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2400	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2401	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2402	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2403	AssertRCReturn(rc, rc);
2404
2405	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2406	/* CR8 writes cause a VM-exit based on TPR threshold. */
2407	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2408	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2409	}
2410	else
2411	{
2412	/*
2413	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2414	* Set this control only for 64-bit guests.
2415	*/
2416	if (pVM->hm.s.fAllow64BitGuests)
2417	{
2418	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2419	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2420	}
2421	}
2422
2423	/* Use MSR-bitmaps if supported by the CPU. */
2424	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2425	{
2426	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2427
2428	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2429	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2430	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2431	AssertRCReturn(rc, rc);
2432
2433	/*
2434	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2435	* automatically using dedicated fields in the VMCS.
2436	*/
2437	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2438	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442
2443	#if HC_ARCH_BITS == 64
2444	/*
2445	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2446	*/
2447	if (pVM->hm.s.fAllow64BitGuests)
2448	{
2449	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453	}
2454	#endif
2455	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2456	}
2457
2458	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2459	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2460	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2461
2462	if ((val & zap) != val)
2463	{
2464	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2465	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2466	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2467	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2468	}
2469
2470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2471	AssertRCReturn(rc, rc);
2472
2473	pVCpu->hm.s.vmx.u32ProcCtls = val;
2474
2475	/*
2476	* Secondary processor-based VM-execution controls.
2477	*/
2478	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2479	{
2480	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2481	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2482
2483	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2484	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2485
2486	if (pVM->hm.s.fNestedPaging)
2487	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2488	else
2489	{
2490	/*
2491	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2492	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2493	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2494	*/
2495	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2496	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2497	}
2498
2499	if (pVM->hm.s.vmx.fVpid)
2500	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2501
2502	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2503	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2504
2505	#if 0
2506	if (pVM->hm.s.fVirtApicRegs)
2507	{
2508	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2509	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2510
2511	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2512	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2513	}
2514	#endif
2515
2516	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2517	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2518	* done dynamically. */
2519	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2520	{
2521	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2522	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2523	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2524	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2525	AssertRCReturn(rc, rc);
2526	}
2527
2528	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2529	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2530
2531	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2532	&& pVM->hm.s.vmx.cPleGapTicks
2533	&& pVM->hm.s.vmx.cPleWindowTicks)
2534	{
2535	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2536
2537	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2538	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2539	AssertRCReturn(rc, rc);
2540	}
2541
2542	if ((val & zap) != val)
2543	{
2544	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2545	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2546	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2547	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2548	}
2549
2550	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2551	AssertRCReturn(rc, rc);
2552
2553	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2554	}
2555	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2556	{
2557	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2558	"available\n"));
2559	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2560	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2561	}
2562
2563	return VINF_SUCCESS;
2564	}
2565
2566
2567	/**
2568	* Sets up miscellaneous (everything other than Pin & Processor-based
2569	* VM-execution) control fields in the VMCS.
2570	*
2571	* @returns VBox status code.
2572	* @param pVM The cross context VM structure.
2573	* @param pVCpu The cross context virtual CPU structure.
2574	*/
2575	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2576	{
2577	NOREF(pVM);
2578	AssertPtr(pVM);
2579	AssertPtr(pVCpu);
2580
2581	int rc = VERR_GENERAL_FAILURE;
2582
2583	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2584	#if 0
2585	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2586	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2587	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2588
2589	/*
2590	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2591	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2592	* We thus use the exception bitmap to control it rather than use both.
2593	*/
2594	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2595	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2596
2597	/** @todo Explore possibility of using IO-bitmaps. */
2598	/* All IO & IOIO instructions cause VM-exits. */
2599	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2600	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2601
2602	/* Initialize the MSR-bitmap area. */
2603	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2604	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2605	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2606	AssertRCReturn(rc, rc);
2607	#endif
2608
2609	/* Setup MSR auto-load/store area. */
2610	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2612	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2613	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614	AssertRCReturn(rc, rc);
2615
2616	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2617	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2618	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2619	AssertRCReturn(rc, rc);
2620
2621	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2622	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2623	AssertRCReturn(rc, rc);
2624
2625	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2626	#if 0
2627	/* Setup debug controls */
2628	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2629	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2630	AssertRCReturn(rc, rc);
2631	#endif
2632
2633	return rc;
2634	}
2635
2636
2637	/**
2638	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2639	*
2640	* We shall setup those exception intercepts that don't change during the
2641	* lifetime of the VM here. The rest are done dynamically while loading the
2642	* guest state.
2643	*
2644	* @returns VBox status code.
2645	* @param pVM The cross context VM structure.
2646	* @param pVCpu The cross context virtual CPU structure.
2647	*/
2648	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2649	{
2650	AssertPtr(pVM);
2651	AssertPtr(pVCpu);
2652
2653	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2654
2655	uint32_t u32XcptBitmap = 0;
2656
2657	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2658	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2659
2660	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2661	and writes, and because recursive #DBs can cause the CPU hang, we must always
2662	intercept #DB. */
2663	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2664
2665	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2666	if (!pVM->hm.s.fNestedPaging)
2667	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2668
2669	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2670	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2671	AssertRCReturn(rc, rc);
2672	return rc;
2673	}
2674
2675
2676	/**
2677	* Sets up the initial guest-state mask. The guest-state mask is consulted
2678	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2679	* for the nested virtualization case (as it would cause a VM-exit).
2680	*
2681	* @param pVCpu The cross context virtual CPU structure.
2682	*/
2683	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2684	{
2685	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2686	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2687	return VINF_SUCCESS;
2688	}
2689
2690
2691	/**
2692	* Does per-VM VT-x initialization.
2693	*
2694	* @returns VBox status code.
2695	* @param pVM The cross context VM structure.
2696	*/
2697	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2698	{
2699	LogFlowFunc(("pVM=%p\n", pVM));
2700
2701	int rc = hmR0VmxStructsAlloc(pVM);
2702	if (RT_FAILURE(rc))
2703	{
2704	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2705	return rc;
2706	}
2707
2708	return VINF_SUCCESS;
2709	}
2710
2711
2712	/**
2713	* Does per-VM VT-x termination.
2714	*
2715	* @returns VBox status code.
2716	* @param pVM The cross context VM structure.
2717	*/
2718	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2719	{
2720	LogFlowFunc(("pVM=%p\n", pVM));
2721
2722	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2723	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2724	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2725	#endif
2726	hmR0VmxStructsFree(pVM);
2727	return VINF_SUCCESS;
2728	}
2729
2730
2731	/**
2732	* Sets up the VM for execution under VT-x.
2733	* This function is only called once per-VM during initialization.
2734	*
2735	* @returns VBox status code.
2736	* @param pVM The cross context VM structure.
2737	*/
2738	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2739	{
2740	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2741	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2742
2743	LogFlowFunc(("pVM=%p\n", pVM));
2744
2745	/*
2746	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2747	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2748	*/
2749	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2750	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2751	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2752	{
2753	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2754	return VERR_INTERNAL_ERROR;
2755	}
2756
2757	/* Initialize these always, see hmR3InitFinalizeR0().*/
2758	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2759	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2760
2761	/* Setup the tagged-TLB flush handlers. */
2762	int rc = hmR0VmxSetupTaggedTlb(pVM);
2763	if (RT_FAILURE(rc))
2764	{
2765	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2766	return rc;
2767	}
2768
2769	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2770	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2771	#if HC_ARCH_BITS == 64
2772	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2773	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2774	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2775	{
2776	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2777	}
2778	#endif
2779
2780	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2781	RTCCUINTREG uHostCR4 = ASMGetCR4();
2782	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2783	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2784
2785	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2786	{
2787	PVMCPU pVCpu = &pVM->aCpus[i];
2788	AssertPtr(pVCpu);
2789	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2790
2791	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2792	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2793
2794	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2795	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2796	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2797
2798	/* Set revision dword at the beginning of the VMCS structure. */
2799	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2800
2801	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2802	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2803	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806	/* Load this VMCS as the current VMCS. */
2807	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2808	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2809	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2810
2811	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2812	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2813	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2814
2815	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2816	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818
2819	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2820	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2821	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2822
2823	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2824	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2825	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2826
2827	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2828	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2829	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2830
2831	#if HC_ARCH_BITS == 32
2832	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2833	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2834	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2835	#endif
2836
2837	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2838	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2839	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2840	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2841
2842	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2843
2844	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2845	}
2846
2847	return VINF_SUCCESS;
2848	}
2849
2850
2851	/**
2852	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2853	* the VMCS.
2854	*
2855	* @returns VBox status code.
2856	* @param pVM The cross context VM structure.
2857	* @param pVCpu The cross context virtual CPU structure.
2858	*/
2859	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2860	{
2861	NOREF(pVM); NOREF(pVCpu);
2862
2863	RTCCUINTREG uReg = ASMGetCR0();
2864	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2865	AssertRCReturn(rc, rc);
2866
2867	uReg = ASMGetCR3();
2868	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2869	AssertRCReturn(rc, rc);
2870
2871	uReg = ASMGetCR4();
2872	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2873	AssertRCReturn(rc, rc);
2874	return rc;
2875	}
2876
2877
2878	#if HC_ARCH_BITS == 64
2879	/**
2880	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2881	* requirements. See hmR0VmxSaveHostSegmentRegs().
2882	*/
2883	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2884	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2885	{ \
2886	bool fValidSelector = true; \
2887	if ((selValue) & X86_SEL_LDT) \
2888	{ \
2889	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2890	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2891	} \
2892	if (fValidSelector) \
2893	{ \
2894	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2895	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2896	} \
2897	(selValue) = 0; \
2898	}
2899	#endif
2900
2901
2902	/**
2903	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2904	* the host-state area in the VMCS.
2905	*
2906	* @returns VBox status code.
2907	* @param pVM The cross context VM structure.
2908	* @param pVCpu The cross context virtual CPU structure.
2909	*/
2910	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2911	{
2912	int rc = VERR_INTERNAL_ERROR_5;
2913
2914	#if HC_ARCH_BITS == 64
2915	/*
2916	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2917	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2918	*
2919	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2920	* Was observed booting Solaris10u10 32-bit guest.
2921	*/
2922	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2923	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2924	{
2925	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2926	pVCpu->idCpu));
2927	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2928	}
2929	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2930	#else
2931	RT_NOREF(pVCpu);
2932	#endif
2933
2934	/*
2935	* Host DS, ES, FS and GS segment registers.
2936	*/
2937	#if HC_ARCH_BITS == 64
2938	RTSEL uSelDS = ASMGetDS();
2939	RTSEL uSelES = ASMGetES();
2940	RTSEL uSelFS = ASMGetFS();
2941	RTSEL uSelGS = ASMGetGS();
2942	#else
2943	RTSEL uSelDS = 0;
2944	RTSEL uSelES = 0;
2945	RTSEL uSelFS = 0;
2946	RTSEL uSelGS = 0;
2947	#endif
2948
2949	/*
2950	* Host CS and SS segment registers.
2951	*/
2952	RTSEL uSelCS = ASMGetCS();
2953	RTSEL uSelSS = ASMGetSS();
2954
2955	/*
2956	* Host TR segment register.
2957	*/
2958	RTSEL uSelTR = ASMGetTR();
2959
2960	#if HC_ARCH_BITS == 64
2961	/*
2962	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2963	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2964	*/
2965	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2966	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2967	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2968	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2969	# undef VMXLOCAL_ADJUST_HOST_SEG
2970	#endif
2971
2972	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2973	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2974	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2975	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2976	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2977	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2978	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2979	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2980	Assert(uSelCS);
2981	Assert(uSelTR);
2982
2983	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2984	#if 0
2985	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2986	Assert(uSelSS != 0);
2987	#endif
2988
2989	/* Write these host selector fields into the host-state area in the VMCS. */
2990	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2991	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2992	#if HC_ARCH_BITS == 64
2993	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2994	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2995	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2996	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2997	#else
2998	NOREF(uSelDS);
2999	NOREF(uSelES);
3000	NOREF(uSelFS);
3001	NOREF(uSelGS);
3002	#endif
3003	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3004	AssertRCReturn(rc, rc);
3005
3006	/*
3007	* Host GDTR and IDTR.
3008	*/
3009	RTGDTR Gdtr;
3010	RTIDTR Idtr;
3011	RT_ZERO(Gdtr);
3012	RT_ZERO(Idtr);
3013	ASMGetGDTR(&Gdtr);
3014	ASMGetIDTR(&Idtr);
3015	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3016	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3017	AssertRCReturn(rc, rc);
3018
3019	#if HC_ARCH_BITS == 64
3020	/*
3021	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3022	* maximum limit (0xffff) on every VM-exit.
3023	*/
3024	if (Gdtr.cbGdt != 0xffff)
3025	{
3026	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3027	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3028	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3029	}
3030
3031	/*
3032	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3033	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3034	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3035	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3036	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3037	* hosts where we are pretty sure it won't cause trouble.
3038	*/
3039	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3040	if (Idtr.cbIdt < 0x0fff)
3041	# else
3042	if (Idtr.cbIdt != 0xffff)
3043	# endif
3044	{
3045	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3046	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3047	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3048	}
3049	#endif
3050
3051	/*
3052	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3053	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3054	*/
3055	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3056	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3057	VERR_VMX_INVALID_HOST_STATE);
3058
3059	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3060	#if HC_ARCH_BITS == 64
3061	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3062
3063	/*
3064	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3065	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3066	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3067	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3068	*
3069	* [1] See Intel spec. 3.5 "System Descriptor Types".
3070	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3071	*/
3072	Assert(pDesc->System.u4Type == 11);
3073	if ( pDesc->System.u16LimitLow != 0x67
3074	\|\| pDesc->System.u4LimitHigh)
3075	{
3076	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3077	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3078	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3079	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3080	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3081
3082	/* Store the GDTR here as we need it while restoring TR. */
3083	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3084	}
3085	#else
3086	NOREF(pVM);
3087	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3088	#endif
3089	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3090	AssertRCReturn(rc, rc);
3091
3092	/*
3093	* Host FS base and GS base.
3094	*/
3095	#if HC_ARCH_BITS == 64
3096	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3097	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3098	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3099	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3100	AssertRCReturn(rc, rc);
3101
3102	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3103	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3104	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3105	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3106	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3107	#endif
3108	return rc;
3109	}
3110
3111
3112	/**
3113	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3114	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3115	* the host after every successful VM-exit.
3116	*
3117	* @returns VBox status code.
3118	* @param pVM The cross context VM structure.
3119	* @param pVCpu The cross context virtual CPU structure.
3120	*
3121	* @remarks No-long-jump zone!!!
3122	*/
3123	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3124	{
3125	NOREF(pVM);
3126
3127	AssertPtr(pVCpu);
3128	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3129
3130	/*
3131	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3132	* rather than swapping them on every VM-entry.
3133	*/
3134	hmR0VmxLazySaveHostMsrs(pVCpu);
3135
3136	/*
3137	* Host Sysenter MSRs.
3138	*/
3139	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3140	#if HC_ARCH_BITS == 32
3141	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3142	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3143	#else
3144	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3145	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3146	#endif
3147	AssertRCReturn(rc, rc);
3148
3149	/*
3150	* Host EFER MSR.
3151	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3152	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3153	*/
3154	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3155	{
3156	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3157	AssertRCReturn(rc, rc);
3158	}
3159
3160	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3161	* hmR0VmxLoadGuestExitCtls() !! */
3162
3163	return rc;
3164	}
3165
3166
3167	/**
3168	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3169	*
3170	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3171	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3172	* hmR0VMxLoadGuestEntryCtls().
3173	*
3174	* @returns true if we need to load guest EFER, false otherwise.
3175	* @param pVCpu The cross context virtual CPU structure.
3176	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3177	* out-of-sync. Make sure to update the required fields
3178	* before using them.
3179	*
3180	* @remarks Requires EFER, CR4.
3181	* @remarks No-long-jump zone!!!
3182	*/
3183	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3184	{
3185	#ifdef HMVMX_ALWAYS_SWAP_EFER
3186	return true;
3187	#endif
3188
3189	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3190	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3191	if (CPUMIsGuestInLongMode(pVCpu))
3192	return false;
3193	#endif
3194
3195	PVM pVM = pVCpu->CTX_SUFF(pVM);
3196	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3197	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3198
3199	/*
3200	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3201	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3202	*/
3203	if ( CPUMIsGuestInLongMode(pVCpu)
3204	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3205	{
3206	return true;
3207	}
3208
3209	/*
3210	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3211	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3212	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3213	*/
3214	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3215	&& (pMixedCtx->cr0 & X86_CR0_PG)
3216	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3217	{
3218	/* Assert that host is PAE capable. */
3219	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3220	return true;
3221	}
3222
3223	/** @todo Check the latest Intel spec. for any other bits,
3224	* like SMEP/SMAP? */
3225	return false;
3226	}
3227
3228
3229	/**
3230	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3231	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3232	* controls".
3233	*
3234	* @returns VBox status code.
3235	* @param pVCpu The cross context virtual CPU structure.
3236	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3237	* out-of-sync. Make sure to update the required fields
3238	* before using them.
3239	*
3240	* @remarks Requires EFER.
3241	* @remarks No-long-jump zone!!!
3242	*/
3243	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3244	{
3245	int rc = VINF_SUCCESS;
3246	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3247	{
3248	PVM pVM = pVCpu->CTX_SUFF(pVM);
3249	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3250	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3251
3252	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3253	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3254
3255	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3256	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3257	{
3258	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3259	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3260	}
3261	else
3262	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3263
3264	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3265	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3266	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3267	{
3268	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3269	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3270	}
3271
3272	/*
3273	* The following should -not- be set (since we're not in SMM mode):
3274	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3275	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3276	*/
3277
3278	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3279	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3280
3281	if ((val & zap) != val)
3282	{
3283	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3284	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3285	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3286	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3287	}
3288
3289	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3290	AssertRCReturn(rc, rc);
3291
3292	pVCpu->hm.s.vmx.u32EntryCtls = val;
3293	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3294	}
3295	return rc;
3296	}
3297
3298
3299	/**
3300	* Sets up the VM-exit controls in the VMCS.
3301	*
3302	* @returns VBox status code.
3303	* @param pVCpu The cross context virtual CPU structure.
3304	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3305	* out-of-sync. Make sure to update the required fields
3306	* before using them.
3307	*
3308	* @remarks Requires EFER.
3309	*/
3310	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3311	{
3312	NOREF(pMixedCtx);
3313
3314	int rc = VINF_SUCCESS;
3315	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3316	{
3317	PVM pVM = pVCpu->CTX_SUFF(pVM);
3318	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3319	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3320
3321	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3322	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3323
3324	/*
3325	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3326	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3327	*/
3328	#if HC_ARCH_BITS == 64
3329	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3330	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3331	#else
3332	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3333	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3334	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3335	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3336	{
3337	/* The switcher returns to long mode, EFER is managed by the switcher. */
3338	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3339	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3340	}
3341	else
3342	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3343	#endif
3344
3345	/* If the newer VMCS fields for managing EFER exists, use it. */
3346	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3347	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3348	{
3349	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3350	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3351	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3352	}
3353
3354	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3355	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3356
3357	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3358	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3359	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3360
3361	if ( pVM->hm.s.vmx.fUsePreemptTimer
3362	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3363	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3364
3365	if ((val & zap) != val)
3366	{
3367	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3368	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3369	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3370	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3371	}
3372
3373	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3374	AssertRCReturn(rc, rc);
3375
3376	pVCpu->hm.s.vmx.u32ExitCtls = val;
3377	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3378	}
3379	return rc;
3380	}
3381
3382
3383	/**
3384	* Sets the TPR threshold in the VMCS.
3385	*
3386	* @returns VBox status code.
3387	* @param pVCpu The cross context virtual CPU structure.
3388	* @param u32TprThreshold The TPR threshold (task-priority class only).
3389	*/
3390	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3391	{
3392	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3393	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3394	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3395	}
3396
3397
3398	/**
3399	* Loads the guest APIC and related state.
3400	*
3401	* @returns VBox status code.
3402	* @param pVCpu The cross context virtual CPU structure.
3403	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3404	* out-of-sync. Make sure to update the required fields
3405	* before using them.
3406	*/
3407	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3408	{
3409	NOREF(pMixedCtx);
3410
3411	int rc = VINF_SUCCESS;
3412	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3413	{
3414	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3415	&& APICIsEnabled(pVCpu))
3416	{
3417	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3418	{
3419	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3420
3421	bool fPendingIntr = false;
3422	uint8_t u8Tpr = 0;
3423	uint8_t u8PendingIntr = 0;
3424	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3425	AssertRCReturn(rc, rc);
3426
3427	/*
3428	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3429	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3430	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3431	*/
3432	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3433	uint32_t u32TprThreshold = 0;
3434	if (fPendingIntr)
3435	{
3436	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3437	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3438	const uint8_t u8TprPriority = u8Tpr >> 4;
3439	if (u8PendingPriority <= u8TprPriority)
3440	u32TprThreshold = u8PendingPriority;
3441	}
3442
3443	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3444	AssertRCReturn(rc, rc);
3445	}
3446
3447	#ifndef IEM_VERIFICATION_MODE_FULL
3448	/* Setup the Virtualized APIC accesses. */
3449	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
3450	{
3451	uint64_t u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
3452	if (u64MsrApicBase != pVCpu->hm.s.vmx.u64MsrApicBase)
3453	{
3454	PVM pVM = pVCpu->CTX_SUFF(pVM);
3455	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
3456	RTGCPHYS GCPhysApicBase;
3457	GCPhysApicBase = u64MsrApicBase;
3458	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3459
3460	/* Unalias any existing mapping. */
3461	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
3462	AssertRCReturn(rc, rc);
3463
3464	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
3465	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
3466	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
3467	AssertRCReturn(rc, rc);
3468
3469	/* Update VMX's cache of the APIC base. */
3470	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
3471	}
3472	}
3473	#endif /* !IEM_VERIFICATION_MODE_FULL */
3474	}
3475	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3476	}
3477
3478	return rc;
3479	}
3480
3481
3482	/**
3483	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3484	*
3485	* @returns Guest's interruptibility-state.
3486	* @param pVCpu The cross context virtual CPU structure.
3487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3488	* out-of-sync. Make sure to update the required fields
3489	* before using them.
3490	*
3491	* @remarks No-long-jump zone!!!
3492	*/
3493	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3494	{
3495	/*
3496	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3497	*/
3498	uint32_t uIntrState = 0;
3499	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3500	{
3501	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3502	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3503	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3504	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3505	{
3506	if (pMixedCtx->eflags.Bits.u1IF)
3507	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3508	else
3509	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3510	}
3511	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3512	{
3513	/*
3514	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3515	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3516	*/
3517	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3518	}
3519	}
3520
3521	/*
3522	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3523	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3524	* setting this would block host-NMIs and IRET will not clear the blocking.
3525	*
3526	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3527	*/
3528	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3529	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3530	{
3531	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3532	}
3533
3534	return uIntrState;
3535	}
3536
3537
3538	/**
3539	* Loads the guest's interruptibility-state into the guest-state area in the
3540	* VMCS.
3541	*
3542	* @returns VBox status code.
3543	* @param pVCpu The cross context virtual CPU structure.
3544	* @param uIntrState The interruptibility-state to set.
3545	*/
3546	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3547	{
3548	NOREF(pVCpu);
3549	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3550	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3551	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3552	AssertRC(rc);
3553	return rc;
3554	}
3555
3556
3557	/**
3558	* Loads the exception intercepts required for guest execution in the VMCS.
3559	*
3560	* @returns VBox status code.
3561	* @param pVCpu The cross context virtual CPU structure.
3562	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563	* out-of-sync. Make sure to update the required fields
3564	* before using them.
3565	*/
3566	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3567	{
3568	NOREF(pMixedCtx);
3569	int rc = VINF_SUCCESS;
3570	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3571	{
3572	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3573	if (pVCpu->hm.s.fGIMTrapXcptUD)
3574	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3575	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3576	else
3577	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3578	#endif
3579
3580	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3581	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3582
3583	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3584	AssertRCReturn(rc, rc);
3585
3586	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3587	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3588	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3589	}
3590	return rc;
3591	}
3592
3593
3594	/**
3595	* Loads the guest's RIP into the guest-state area in the VMCS.
3596	*
3597	* @returns VBox status code.
3598	* @param pVCpu The cross context virtual CPU structure.
3599	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3600	* out-of-sync. Make sure to update the required fields
3601	* before using them.
3602	*
3603	* @remarks No-long-jump zone!!!
3604	*/
3605	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3606	{
3607	int rc = VINF_SUCCESS;
3608	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3609	{
3610	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3611	AssertRCReturn(rc, rc);
3612
3613	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3614	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3615	HMCPU_CF_VALUE(pVCpu)));
3616	}
3617	return rc;
3618	}
3619
3620
3621	/**
3622	* Loads the guest's RSP into the guest-state area in the VMCS.
3623	*
3624	* @returns VBox status code.
3625	* @param pVCpu The cross context virtual CPU structure.
3626	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3627	* out-of-sync. Make sure to update the required fields
3628	* before using them.
3629	*
3630	* @remarks No-long-jump zone!!!
3631	*/
3632	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3633	{
3634	int rc = VINF_SUCCESS;
3635	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3636	{
3637	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3638	AssertRCReturn(rc, rc);
3639
3640	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3641	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3642	}
3643	return rc;
3644	}
3645
3646
3647	/**
3648	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3649	*
3650	* @returns VBox status code.
3651	* @param pVCpu The cross context virtual CPU structure.
3652	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3653	* out-of-sync. Make sure to update the required fields
3654	* before using them.
3655	*
3656	* @remarks No-long-jump zone!!!
3657	*/
3658	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3659	{
3660	int rc = VINF_SUCCESS;
3661	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3662	{
3663	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3664	Let us assert it as such and use 32-bit VMWRITE. */
3665	Assert(!(pMixedCtx->rflags.u64 >> 32));
3666	X86EFLAGS Eflags = pMixedCtx->eflags;
3667	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3668	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3669	* These will never be cleared/set, unless some other part of the VMM
3670	* code is buggy - in which case we're better of finding and fixing
3671	* those bugs than hiding them. */
3672	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3673	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3674	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3675	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3676
3677	/*
3678	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3679	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3680	*/
3681	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3682	{
3683	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3684	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3685	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3686	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3687	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3688	}
3689
3690	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3691	AssertRCReturn(rc, rc);
3692
3693	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3694	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3695	}
3696	return rc;
3697	}
3698
3699
3700	/**
3701	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3702	*
3703	* @returns VBox status code.
3704	* @param pVCpu The cross context virtual CPU structure.
3705	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3706	* out-of-sync. Make sure to update the required fields
3707	* before using them.
3708	*
3709	* @remarks No-long-jump zone!!!
3710	*/
3711	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3712	{
3713	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3714	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3715	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3716	AssertRCReturn(rc, rc);
3717	return rc;
3718	}
3719
3720
3721	/**
3722	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3723	* CR0 is partially shared with the host and we have to consider the FPU bits.
3724	*
3725	* @returns VBox status code.
3726	* @param pVCpu The cross context virtual CPU structure.
3727	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3728	* out-of-sync. Make sure to update the required fields
3729	* before using them.
3730	*
3731	* @remarks No-long-jump zone!!!
3732	*/
3733	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3734	{
3735	/*
3736	* Guest CR0.
3737	* Guest FPU.
3738	*/
3739	int rc = VINF_SUCCESS;
3740	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3741	{
3742	Assert(!(pMixedCtx->cr0 >> 32));
3743	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3744	PVM pVM = pVCpu->CTX_SUFF(pVM);
3745
3746	/* The guest's view (read access) of its CR0 is unblemished. */
3747	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3748	AssertRCReturn(rc, rc);
3749	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3750
3751	/* Setup VT-x's view of the guest CR0. */
3752	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3753	if (pVM->hm.s.fNestedPaging)
3754	{
3755	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3756	{
3757	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3758	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3759	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3760	}
3761	else
3762	{
3763	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3764	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3765	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3766	}
3767
3768	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3769	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3770	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3771
3772	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3773	AssertRCReturn(rc, rc);
3774	}
3775	else
3776	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3777
3778	/*
3779	* Guest FPU bits.
3780	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3781	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3782	*/
3783	u32GuestCR0 \|= X86_CR0_NE;
3784	bool fInterceptNM = false;
3785	if (CPUMIsGuestFPUStateActive(pVCpu))
3786	{
3787	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3788	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3789	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3790	}
3791	else
3792	{
3793	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3794	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3795	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3796	}
3797
3798	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3799	bool fInterceptMF = false;
3800	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3801	fInterceptMF = true;
3802
3803	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3804	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3805	{
3806	Assert(PDMVmmDevHeapIsEnabled(pVM));
3807	Assert(pVM->hm.s.vmx.pRealModeTSS);
3808	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3809	fInterceptNM = true;
3810	fInterceptMF = true;
3811	}
3812	else
3813	{
3814	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3815	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3816	}
3817	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3818
3819	if (fInterceptNM)
3820	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3821	else
3822	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3823
3824	if (fInterceptMF)
3825	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3826	else
3827	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3828
3829	/* Additional intercepts for debugging, define these yourself explicitly. */
3830	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3831	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3832	\| RT_BIT(X86_XCPT_BP)
3833	\| RT_BIT(X86_XCPT_DE)
3834	\| RT_BIT(X86_XCPT_NM)
3835	\| RT_BIT(X86_XCPT_TS)
3836	\| RT_BIT(X86_XCPT_UD)
3837	\| RT_BIT(X86_XCPT_NP)
3838	\| RT_BIT(X86_XCPT_SS)
3839	\| RT_BIT(X86_XCPT_GP)
3840	\| RT_BIT(X86_XCPT_PF)
3841	\| RT_BIT(X86_XCPT_MF)
3842	;
3843	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3844	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3845	#endif
3846
3847	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3848
3849	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3850	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3851	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3852	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3853	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3854	else
3855	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3856
3857	u32GuestCR0 \|= uSetCR0;
3858	u32GuestCR0 &= uZapCR0;
3859	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3860
3861	/* Write VT-x's view of the guest CR0 into the VMCS. */
3862	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3863	AssertRCReturn(rc, rc);
3864	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3865	uZapCR0));
3866
3867	/*
3868	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3869	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3870	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3871	*/
3872	uint32_t u32CR0Mask = 0;
3873	u32CR0Mask = X86_CR0_PE
3874	\| X86_CR0_NE
3875	\| X86_CR0_WP
3876	\| X86_CR0_PG
3877	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3878	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3879	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3880
3881	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3882	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3883	* and @bugref{6944}. */
3884	#if 0
3885	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3886	u32CR0Mask &= ~X86_CR0_PE;
3887	#endif
3888	if (pVM->hm.s.fNestedPaging)
3889	u32CR0Mask &= ~X86_CR0_WP;
3890
3891	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3892	if (fInterceptNM)
3893	{
3894	u32CR0Mask \|= X86_CR0_TS
3895	\| X86_CR0_MP;
3896	}
3897
3898	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3899	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3900	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3901	AssertRCReturn(rc, rc);
3902	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3903
3904	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3905	}
3906	return rc;
3907	}
3908
3909
3910	/**
3911	* Loads the guest control registers (CR3, CR4) into the guest-state area
3912	* in the VMCS.
3913	*
3914	* @returns VBox strict status code.
3915	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3916	* without unrestricted guest access and the VMMDev is not presently
3917	* mapped (e.g. EFI32).
3918	*
3919	* @param pVCpu The cross context virtual CPU structure.
3920	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3921	* out-of-sync. Make sure to update the required fields
3922	* before using them.
3923	*
3924	* @remarks No-long-jump zone!!!
3925	*/
3926	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3927	{
3928	int rc = VINF_SUCCESS;
3929	PVM pVM = pVCpu->CTX_SUFF(pVM);
3930
3931	/*
3932	* Guest CR2.
3933	* It's always loaded in the assembler code. Nothing to do here.
3934	*/
3935
3936	/*
3937	* Guest CR3.
3938	*/
3939	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3940	{
3941	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3942	if (pVM->hm.s.fNestedPaging)
3943	{
3944	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3945
3946	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3947	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3948	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3949	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3950
3951	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3952	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3953	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3954
3955	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3956	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3957	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3958	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3959	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3960	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3961	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3962
3963	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3964	AssertRCReturn(rc, rc);
3965	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3966
3967	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3968	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3969	{
3970	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3971	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3972	{
3973	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3974	AssertRCReturn(rc, rc);
3975	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3976	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3977	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3978	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3979	AssertRCReturn(rc, rc);
3980	}
3981
3982	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3983	have Unrestricted Execution to handle the guest when it's not using paging. */
3984	GCPhysGuestCR3 = pMixedCtx->cr3;
3985	}
3986	else
3987	{
3988	/*
3989	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3990	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3991	* EPT takes care of translating it to host-physical addresses.
3992	*/
3993	RTGCPHYS GCPhys;
3994	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3995
3996	/* We obtain it here every time as the guest could have relocated this PCI region. */
3997	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3998	if (RT_SUCCESS(rc))
3999	{ /* likely */ }
4000	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4001	{
4002	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4003	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4004	}
4005	else
4006	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4007
4008	GCPhysGuestCR3 = GCPhys;
4009	}
4010
4011	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4012	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4013	}
4014	else
4015	{
4016	/* Non-nested paging case, just use the hypervisor's CR3. */
4017	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4018
4019	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4020	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4021	}
4022	AssertRCReturn(rc, rc);
4023
4024	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4025	}
4026
4027	/*
4028	* Guest CR4.
4029	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4030	*/
4031	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4032	{
4033	Assert(!(pMixedCtx->cr4 >> 32));
4034	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4035
4036	/* The guest's view of its CR4 is unblemished. */
4037	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4038	AssertRCReturn(rc, rc);
4039	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4040
4041	/* Setup VT-x's view of the guest CR4. */
4042	/*
4043	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4044	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4045	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4046	*/
4047	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4048	{
4049	Assert(pVM->hm.s.vmx.pRealModeTSS);
4050	Assert(PDMVmmDevHeapIsEnabled(pVM));
4051	u32GuestCR4 &= ~X86_CR4_VME;
4052	}
4053
4054	if (pVM->hm.s.fNestedPaging)
4055	{
4056	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4057	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4058	{
4059	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4060	u32GuestCR4 \|= X86_CR4_PSE;
4061	/* Our identity mapping is a 32-bit page directory. */
4062	u32GuestCR4 &= ~X86_CR4_PAE;
4063	}
4064	/* else use guest CR4.*/
4065	}
4066	else
4067	{
4068	/*
4069	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4070	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4071	*/
4072	switch (pVCpu->hm.s.enmShadowMode)
4073	{
4074	case PGMMODE_REAL: /* Real-mode. */
4075	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4076	case PGMMODE_32_BIT: /* 32-bit paging. */
4077	{
4078	u32GuestCR4 &= ~X86_CR4_PAE;
4079	break;
4080	}
4081
4082	case PGMMODE_PAE: /* PAE paging. */
4083	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4084	{
4085	u32GuestCR4 \|= X86_CR4_PAE;
4086	break;
4087	}
4088
4089	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4090	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4091	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4092	break;
4093	#endif
4094	default:
4095	AssertFailed();
4096	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4097	}
4098	}
4099
4100	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4101	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4102	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4103	u32GuestCR4 \|= uSetCR4;
4104	u32GuestCR4 &= uZapCR4;
4105
4106	/* Write VT-x's view of the guest CR4 into the VMCS. */
4107	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4108	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4109	AssertRCReturn(rc, rc);
4110
4111	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4112	uint32_t u32CR4Mask = X86_CR4_VME
4113	\| X86_CR4_PAE
4114	\| X86_CR4_PGE
4115	\| X86_CR4_PSE
4116	\| X86_CR4_VMXE;
4117	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4118	u32CR4Mask \|= X86_CR4_OSXSAVE;
4119	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4120	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4121	AssertRCReturn(rc, rc);
4122
4123	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4124	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4125
4126	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4127	}
4128	return rc;
4129	}
4130
4131
4132	/**
4133	* Loads the guest debug registers into the guest-state area in the VMCS.
4134	*
4135	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4136	*
4137	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4138	*
4139	* @returns VBox status code.
4140	* @param pVCpu The cross context virtual CPU structure.
4141	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4142	* out-of-sync. Make sure to update the required fields
4143	* before using them.
4144	*
4145	* @remarks No-long-jump zone!!!
4146	*/
4147	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4148	{
4149	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4150	return VINF_SUCCESS;
4151
4152	#ifdef VBOX_STRICT
4153	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4154	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4155	{
4156	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4157	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4158	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4159	}
4160	#endif
4161
4162	int rc;
4163	PVM pVM = pVCpu->CTX_SUFF(pVM);
4164	bool fSteppingDB = false;
4165	bool fInterceptMovDRx = false;
4166	if (pVCpu->hm.s.fSingleInstruction)
4167	{
4168	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4169	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4170	{
4171	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4172	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4173	AssertRCReturn(rc, rc);
4174	Assert(fSteppingDB == false);
4175	}
4176	else
4177	{
4178	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4179	pVCpu->hm.s.fClearTrapFlag = true;
4180	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4181	fSteppingDB = true;
4182	}
4183	}
4184
4185	if ( fSteppingDB
4186	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4187	{
4188	/*
4189	* Use the combined guest and host DRx values found in the hypervisor
4190	* register set because the debugger has breakpoints active or someone
4191	* is single stepping on the host side without a monitor trap flag.
4192	*
4193	* Note! DBGF expects a clean DR6 state before executing guest code.
4194	*/
4195	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4196	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4197	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4198	{
4199	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4200	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4201	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4202	}
4203	else
4204	#endif
4205	if (!CPUMIsHyperDebugStateActive(pVCpu))
4206	{
4207	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4208	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4209	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4210	}
4211
4212	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4213	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4214	AssertRCReturn(rc, rc);
4215
4216	pVCpu->hm.s.fUsingHyperDR7 = true;
4217	fInterceptMovDRx = true;
4218	}
4219	else
4220	{
4221	/*
4222	* If the guest has enabled debug registers, we need to load them prior to
4223	* executing guest code so they'll trigger at the right time.
4224	*/
4225	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4226	{
4227	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4228	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4229	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4230	{
4231	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4232	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4233	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4234	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4235	}
4236	else
4237	#endif
4238	if (!CPUMIsGuestDebugStateActive(pVCpu))
4239	{
4240	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4241	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4242	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4243	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4244	}
4245	Assert(!fInterceptMovDRx);
4246	}
4247	/*
4248	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4249	* must intercept #DB in order to maintain a correct DR6 guest value, and
4250	* because we need to intercept it to prevent nested #DBs from hanging the
4251	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4252	*/
4253	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4254	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4255	&& !CPUMIsGuestDebugStateActive(pVCpu))
4256	#else
4257	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4258	#endif
4259	{
4260	fInterceptMovDRx = true;
4261	}
4262
4263	/* Update guest DR7. */
4264	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4265	AssertRCReturn(rc, rc);
4266
4267	pVCpu->hm.s.fUsingHyperDR7 = false;
4268	}
4269
4270	/*
4271	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4272	*/
4273	if (fInterceptMovDRx)
4274	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4275	else
4276	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4277	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4278	AssertRCReturn(rc, rc);
4279
4280	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4281	return VINF_SUCCESS;
4282	}
4283
4284
4285	#ifdef VBOX_STRICT
4286	/**
4287	* Strict function to validate segment registers.
4288	*
4289	* @remarks ASSUMES CR0 is up to date.
4290	*/
4291	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4292	{
4293	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4294	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4295	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4296	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4297	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4298	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4299	{
4300	/* Protected mode checks */
4301	/* CS */
4302	Assert(pCtx->cs.Attr.n.u1Present);
4303	Assert(!(pCtx->cs.Attr.u & 0xf00));
4304	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4305	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4306	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4307	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4308	\|\| (pCtx->cs.Attr.n.u1Granularity));
4309	/* CS cannot be loaded with NULL in protected mode. */
4310	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4311	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4312	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4313	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4314	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4315	else
4316	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4317	/* SS */
4318	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4319	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4320	if ( !(pCtx->cr0 & X86_CR0_PE)
4321	\|\| pCtx->cs.Attr.n.u4Type == 3)
4322	{
4323	Assert(!pCtx->ss.Attr.n.u2Dpl);
4324	}
4325	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4326	{
4327	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4328	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4329	Assert(pCtx->ss.Attr.n.u1Present);
4330	Assert(!(pCtx->ss.Attr.u & 0xf00));
4331	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4332	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4333	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4334	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4335	\|\| (pCtx->ss.Attr.n.u1Granularity));
4336	}
4337	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4338	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4339	{
4340	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4341	Assert(pCtx->ds.Attr.n.u1Present);
4342	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4343	Assert(!(pCtx->ds.Attr.u & 0xf00));
4344	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4345	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4346	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4347	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4348	\|\| (pCtx->ds.Attr.n.u1Granularity));
4349	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4350	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4351	}
4352	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4353	{
4354	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4355	Assert(pCtx->es.Attr.n.u1Present);
4356	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4357	Assert(!(pCtx->es.Attr.u & 0xf00));
4358	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4359	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4360	\|\| !(pCtx->es.Attr.n.u1Granularity));
4361	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4362	\|\| (pCtx->es.Attr.n.u1Granularity));
4363	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4364	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4365	}
4366	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4367	{
4368	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4369	Assert(pCtx->fs.Attr.n.u1Present);
4370	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4371	Assert(!(pCtx->fs.Attr.u & 0xf00));
4372	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4373	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4374	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4375	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4376	\|\| (pCtx->fs.Attr.n.u1Granularity));
4377	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4378	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4379	}
4380	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4381	{
4382	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4383	Assert(pCtx->gs.Attr.n.u1Present);
4384	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4385	Assert(!(pCtx->gs.Attr.u & 0xf00));
4386	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4387	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4388	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4389	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4390	\|\| (pCtx->gs.Attr.n.u1Granularity));
4391	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4392	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4393	}
4394	/* 64-bit capable CPUs. */
4395	# if HC_ARCH_BITS == 64
4396	Assert(!(pCtx->cs.u64Base >> 32));
4397	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4398	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4399	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4400	# endif
4401	}
4402	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4403	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4404	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4405	{
4406	/* Real and v86 mode checks. */
4407	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4408	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4409	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4410	{
4411	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4412	}
4413	else
4414	{
4415	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4416	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4417	}
4418
4419	/* CS */
4420	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4421	Assert(pCtx->cs.u32Limit == 0xffff);
4422	Assert(u32CSAttr == 0xf3);
4423	/* SS */
4424	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4425	Assert(pCtx->ss.u32Limit == 0xffff);
4426	Assert(u32SSAttr == 0xf3);
4427	/* DS */
4428	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4429	Assert(pCtx->ds.u32Limit == 0xffff);
4430	Assert(u32DSAttr == 0xf3);
4431	/* ES */
4432	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4433	Assert(pCtx->es.u32Limit == 0xffff);
4434	Assert(u32ESAttr == 0xf3);
4435	/* FS */
4436	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4437	Assert(pCtx->fs.u32Limit == 0xffff);
4438	Assert(u32FSAttr == 0xf3);
4439	/* GS */
4440	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4441	Assert(pCtx->gs.u32Limit == 0xffff);
4442	Assert(u32GSAttr == 0xf3);
4443	/* 64-bit capable CPUs. */
4444	# if HC_ARCH_BITS == 64
4445	Assert(!(pCtx->cs.u64Base >> 32));
4446	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4447	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4448	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4449	# endif
4450	}
4451	}
4452	#endif /* VBOX_STRICT */
4453
4454
4455	/**
4456	* Writes a guest segment register into the guest-state area in the VMCS.
4457	*
4458	* @returns VBox status code.
4459	* @param pVCpu The cross context virtual CPU structure.
4460	* @param idxSel Index of the selector in the VMCS.
4461	* @param idxLimit Index of the segment limit in the VMCS.
4462	* @param idxBase Index of the segment base in the VMCS.
4463	* @param idxAccess Index of the access rights of the segment in the VMCS.
4464	* @param pSelReg Pointer to the segment selector.
4465	*
4466	* @remarks No-long-jump zone!!!
4467	*/
4468	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4469	uint32_t idxAccess, PCPUMSELREG pSelReg)
4470	{
4471	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4472	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4473	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4474	AssertRCReturn(rc, rc);
4475
4476	uint32_t u32Access = pSelReg->Attr.u;
4477	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4478	{
4479	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4480	u32Access = 0xf3;
4481	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4482	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4483	}
4484	else
4485	{
4486	/*
4487	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4488	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4489	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4490	* loaded in protected-mode have their attribute as 0.
4491	*/
4492	if (!u32Access)
4493	u32Access = X86DESCATTR_UNUSABLE;
4494	}
4495
4496	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4497	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4498	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4499
4500	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4501	AssertRCReturn(rc, rc);
4502	return rc;
4503	}
4504
4505
4506	/**
4507	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4508	* into the guest-state area in the VMCS.
4509	*
4510	* @returns VBox status code.
4511	* @param pVCpu The cross context virtual CPU structure.
4512	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4513	* out-of-sync. Make sure to update the required fields
4514	* before using them.
4515	*
4516	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4517	* @remarks No-long-jump zone!!!
4518	*/
4519	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4520	{
4521	int rc = VERR_INTERNAL_ERROR_5;
4522	PVM pVM = pVCpu->CTX_SUFF(pVM);
4523
4524	/*
4525	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4526	*/
4527	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4528	{
4529	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4530	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4531	{
4532	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4533	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4534	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4535	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4536	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4537	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4538	}
4539
4540	#ifdef VBOX_WITH_REM
4541	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4542	{
4543	Assert(pVM->hm.s.vmx.pRealModeTSS);
4544	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4545	if ( pVCpu->hm.s.vmx.fWasInRealMode
4546	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4547	{
4548	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4549	in real-mode (e.g. OpenBSD 4.0) */
4550	REMFlushTBs(pVM);
4551	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4552	pVCpu->hm.s.vmx.fWasInRealMode = false;
4553	}
4554	}
4555	#endif
4556	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4557	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4558	AssertRCReturn(rc, rc);
4559	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4560	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4561	AssertRCReturn(rc, rc);
4562	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4563	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4564	AssertRCReturn(rc, rc);
4565	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4566	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4567	AssertRCReturn(rc, rc);
4568	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4569	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4570	AssertRCReturn(rc, rc);
4571	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4572	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4573	AssertRCReturn(rc, rc);
4574
4575	#ifdef VBOX_STRICT
4576	/* Validate. */
4577	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4578	#endif
4579
4580	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4581	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4582	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4583	}
4584
4585	/*
4586	* Guest TR.
4587	*/
4588	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4589	{
4590	/*
4591	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4592	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4593	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4594	*/
4595	uint16_t u16Sel = 0;
4596	uint32_t u32Limit = 0;
4597	uint64_t u64Base = 0;
4598	uint32_t u32AccessRights = 0;
4599
4600	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4601	{
4602	u16Sel = pMixedCtx->tr.Sel;
4603	u32Limit = pMixedCtx->tr.u32Limit;
4604	u64Base = pMixedCtx->tr.u64Base;
4605	u32AccessRights = pMixedCtx->tr.Attr.u;
4606	}
4607	else
4608	{
4609	Assert(pVM->hm.s.vmx.pRealModeTSS);
4610	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4611
4612	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4613	RTGCPHYS GCPhys;
4614	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4615	AssertRCReturn(rc, rc);
4616
4617	X86DESCATTR DescAttr;
4618	DescAttr.u = 0;
4619	DescAttr.n.u1Present = 1;
4620	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4621
4622	u16Sel = 0;
4623	u32Limit = HM_VTX_TSS_SIZE;
4624	u64Base = GCPhys; /* in real-mode phys = virt. */
4625	u32AccessRights = DescAttr.u;
4626	}
4627
4628	/* Validate. */
4629	Assert(!(u16Sel & RT_BIT(2)));
4630	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4631	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4632	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4633	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4634	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4635	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4636	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4637	Assert( (u32Limit & 0xfff) == 0xfff
4638	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4639	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4640	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4641
4642	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4643	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4644	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4645	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4646	AssertRCReturn(rc, rc);
4647
4648	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4649	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4650	}
4651
4652	/*
4653	* Guest GDTR.
4654	*/
4655	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4656	{
4657	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4658	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4659	AssertRCReturn(rc, rc);
4660
4661	/* Validate. */
4662	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4663
4664	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4665	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4666	}
4667
4668	/*
4669	* Guest LDTR.
4670	*/
4671	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4672	{
4673	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4674	uint32_t u32Access = 0;
4675	if (!pMixedCtx->ldtr.Attr.u)
4676	u32Access = X86DESCATTR_UNUSABLE;
4677	else
4678	u32Access = pMixedCtx->ldtr.Attr.u;
4679
4680	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4681	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4682	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4683	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4684	AssertRCReturn(rc, rc);
4685
4686	/* Validate. */
4687	if (!(u32Access & X86DESCATTR_UNUSABLE))
4688	{
4689	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4690	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4691	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4692	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4693	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4694	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4695	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4696	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4697	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4698	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4699	}
4700
4701	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4702	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4703	}
4704
4705	/*
4706	* Guest IDTR.
4707	*/
4708	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4709	{
4710	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4711	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4712	AssertRCReturn(rc, rc);
4713
4714	/* Validate. */
4715	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4716
4717	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4718	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4719	}
4720
4721	return VINF_SUCCESS;
4722	}
4723
4724
4725	/**
4726	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4727	* areas.
4728	*
4729	* These MSRs will automatically be loaded to the host CPU on every successful
4730	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4731	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4732	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4733	*
4734	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4735	*
4736	* @returns VBox status code.
4737	* @param pVCpu The cross context virtual CPU structure.
4738	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4739	* out-of-sync. Make sure to update the required fields
4740	* before using them.
4741	*
4742	* @remarks No-long-jump zone!!!
4743	*/
4744	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4745	{
4746	AssertPtr(pVCpu);
4747	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4748
4749	/*
4750	* MSRs that we use the auto-load/store MSR area in the VMCS.
4751	*/
4752	PVM pVM = pVCpu->CTX_SUFF(pVM);
4753	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4754	{
4755	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4756	#if HC_ARCH_BITS == 32
4757	if (pVM->hm.s.fAllow64BitGuests)
4758	{
4759	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4760	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4761	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4762	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4763	AssertRCReturn(rc, rc);
4764	# ifdef LOG_ENABLED
4765	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4766	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4767	{
4768	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4769	pMsr->u64Value));
4770	}
4771	# endif
4772	}
4773	#endif
4774	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4775	}
4776
4777	/*
4778	* Guest Sysenter MSRs.
4779	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4780	* VM-exits on WRMSRs for these MSRs.
4781	*/
4782	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4783	{
4784	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4785	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4786	}
4787
4788	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4789	{
4790	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4791	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4792	}
4793
4794	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4795	{
4796	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4797	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4798	}
4799
4800	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4801	{
4802	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4803	{
4804	/*
4805	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4806	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4807	*/
4808	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4809	{
4810	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4811	AssertRCReturn(rc,rc);
4812	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4813	}
4814	else
4815	{
4816	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4817	NULL /* pfAddedAndUpdated */);
4818	AssertRCReturn(rc, rc);
4819
4820	/* We need to intercept reads too, see @bugref{7386#c16}. */
4821	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4822	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4823	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4824	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4825	}
4826	}
4827	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4828	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4829	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4830	}
4831
4832	return VINF_SUCCESS;
4833	}
4834
4835
4836	/**
4837	* Loads the guest activity state into the guest-state area in the VMCS.
4838	*
4839	* @returns VBox status code.
4840	* @param pVCpu The cross context virtual CPU structure.
4841	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4842	* out-of-sync. Make sure to update the required fields
4843	* before using them.
4844	*
4845	* @remarks No-long-jump zone!!!
4846	*/
4847	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4848	{
4849	NOREF(pMixedCtx);
4850	/** @todo See if we can make use of other states, e.g.
4851	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4852	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4853	{
4854	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4855	AssertRCReturn(rc, rc);
4856
4857	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4858	}
4859	return VINF_SUCCESS;
4860	}
4861
4862
4863	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4864	/**
4865	* Check if guest state allows safe use of 32-bit switcher again.
4866	*
4867	* Segment bases and protected mode structures must be 32-bit addressable
4868	* because the 32-bit switcher will ignore high dword when writing these VMCS
4869	* fields. See @bugref{8432} for details.
4870	*
4871	* @returns true if safe, false if must continue to use the 64-bit switcher.
4872	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4873	* out-of-sync. Make sure to update the required fields
4874	* before using them.
4875	*
4876	* @remarks No-long-jump zone!!!
4877	*/
4878	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4879	{
4880	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4881	return false;
4882	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4883	return false;
4884	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4885	return false;
4886	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4887	return false;
4888	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4889	return false;
4890	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4891	return false;
4892	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4893	return false;
4894	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4895	return false;
4896	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4897	return false;
4898	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4899	return false;
4900	/* All good, bases are 32-bit. */
4901	return true;
4902	}
4903	#endif
4904
4905
4906	/**
4907	* Sets up the appropriate function to run guest code.
4908	*
4909	* @returns VBox status code.
4910	* @param pVCpu The cross context virtual CPU structure.
4911	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4912	* out-of-sync. Make sure to update the required fields
4913	* before using them.
4914	*
4915	* @remarks No-long-jump zone!!!
4916	*/
4917	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4918	{
4919	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4920	{
4921	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4922	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4923	#endif
4924	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4925	#if HC_ARCH_BITS == 32
4926	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4927	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4928	{
4929	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4930	{
4931	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4932	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4933	\| HM_CHANGED_VMX_ENTRY_CTLS
4934	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4935	}
4936	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4937
4938	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4939	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4940	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4941	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4942	}
4943	#else
4944	/* 64-bit host. */
4945	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4946	#endif
4947	}
4948	else
4949	{
4950	/* Guest is not in long mode, use the 32-bit handler. */
4951	#if HC_ARCH_BITS == 32
4952	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4953	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4954	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4955	{
4956	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4957	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4958	\| HM_CHANGED_VMX_ENTRY_CTLS
4959	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4960	}
4961	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4962	/*
4963	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4964	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4965	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4966	* the much faster 32-bit switcher again.
4967	*/
4968	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4969	{
4970	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4971	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4972	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4973	}
4974	else
4975	{
4976	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4977	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4978	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4979	{
4980	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4981	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4982	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4983	\| HM_CHANGED_VMX_ENTRY_CTLS
4984	\| HM_CHANGED_VMX_EXIT_CTLS
4985	\| HM_CHANGED_HOST_CONTEXT);
4986	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4987	}
4988	}
4989	# else
4990	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4991	# endif
4992	#else
4993	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4994	#endif
4995	}
4996	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4997	return VINF_SUCCESS;
4998	}
4999
5000
5001	/**
5002	* Wrapper for running the guest code in VT-x.
5003	*
5004	* @returns VBox status code, no informational status codes.
5005	* @param pVM The cross context VM structure.
5006	* @param pVCpu The cross context virtual CPU structure.
5007	* @param pCtx Pointer to the guest-CPU context.
5008	*
5009	* @remarks No-long-jump zone!!!
5010	*/
5011	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5012	{
5013	/*
5014	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5015	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5016	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5017	*/
5018	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5019	/** @todo Add stats for resume vs launch. */
5020	#ifdef VBOX_WITH_KERNEL_USING_XMM
5021	int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5022	#else
5023	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5024	#endif
5025	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5026	return rc;
5027	}
5028
5029
5030	/**
5031	* Reports world-switch error and dumps some useful debug info.
5032	*
5033	* @param pVM The cross context VM structure.
5034	* @param pVCpu The cross context virtual CPU structure.
5035	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5036	* @param pCtx Pointer to the guest-CPU context.
5037	* @param pVmxTransient Pointer to the VMX transient structure (only
5038	* exitReason updated).
5039	*/
5040	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5041	{
5042	Assert(pVM);
5043	Assert(pVCpu);
5044	Assert(pCtx);
5045	Assert(pVmxTransient);
5046	HMVMX_ASSERT_PREEMPT_SAFE();
5047
5048	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5049	switch (rcVMRun)
5050	{
5051	case VERR_VMX_INVALID_VMXON_PTR:
5052	AssertFailed();
5053	break;
5054	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5055	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5056	{
5057	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5058	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5059	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5060	AssertRC(rc);
5061
5062	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5063	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5064	Cannot do it here as we may have been long preempted. */
5065
5066	#ifdef VBOX_STRICT
5067	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5068	pVmxTransient->uExitReason));
5069	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5070	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5071	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5072	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5073	else
5074	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5075	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5076	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5077
5078	/* VMX control bits. */
5079	uint32_t u32Val;
5080	uint64_t u64Val;
5081	RTHCUINTREG uHCReg;
5082	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5083	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5085	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5086	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5087	{
5088	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5089	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5090	}
5091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5092	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5094	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5096	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5098	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5100	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5102	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5104	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5106	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5108	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5110	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5112	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5114	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5116	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5117	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5118	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5119	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5120	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5121	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5122	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5123	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5124	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5125	if (pVM->hm.s.fNestedPaging)
5126	{
5127	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5128	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5129	}
5130
5131	/* Guest bits. */
5132	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5133	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5134	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5135	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5136	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5137	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5138	if (pVM->hm.s.vmx.fVpid)
5139	{
5140	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5141	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5142	}
5143
5144	/* Host bits. */
5145	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5146	Log4(("Host CR0 %#RHr\n", uHCReg));
5147	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5148	Log4(("Host CR3 %#RHr\n", uHCReg));
5149	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5150	Log4(("Host CR4 %#RHr\n", uHCReg));
5151
5152	RTGDTR HostGdtr;
5153	PCX86DESCHC pDesc;
5154	ASMGetGDTR(&HostGdtr);
5155	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5156	Log4(("Host CS %#08x\n", u32Val));
5157	if (u32Val < HostGdtr.cbGdt)
5158	{
5159	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5160	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5161	}
5162
5163	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5164	Log4(("Host DS %#08x\n", u32Val));
5165	if (u32Val < HostGdtr.cbGdt)
5166	{
5167	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5168	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5169	}
5170
5171	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5172	Log4(("Host ES %#08x\n", u32Val));
5173	if (u32Val < HostGdtr.cbGdt)
5174	{
5175	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5176	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5177	}
5178
5179	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5180	Log4(("Host FS %#08x\n", u32Val));
5181	if (u32Val < HostGdtr.cbGdt)
5182	{
5183	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5184	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5185	}
5186
5187	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5188	Log4(("Host GS %#08x\n", u32Val));
5189	if (u32Val < HostGdtr.cbGdt)
5190	{
5191	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5192	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5193	}
5194
5195	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5196	Log4(("Host SS %#08x\n", u32Val));
5197	if (u32Val < HostGdtr.cbGdt)
5198	{
5199	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5200	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5201	}
5202
5203	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5204	Log4(("Host TR %#08x\n", u32Val));
5205	if (u32Val < HostGdtr.cbGdt)
5206	{
5207	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5208	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5209	}
5210
5211	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5212	Log4(("Host TR Base %#RHv\n", uHCReg));
5213	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5214	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5215	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5216	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5217	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5218	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5219	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5220	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5221	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5222	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5224	Log4(("Host RSP %#RHv\n", uHCReg));
5225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5226	Log4(("Host RIP %#RHv\n", uHCReg));
5227	# if HC_ARCH_BITS == 64
5228	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5229	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5230	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5231	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5232	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5233	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5234	# endif
5235	#endif /* VBOX_STRICT */
5236	break;
5237	}
5238
5239	default:
5240	/* Impossible */
5241	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5242	break;
5243	}
5244	NOREF(pVM); NOREF(pCtx);
5245	}
5246
5247
5248	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5249	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5250	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5251	#endif
5252	#ifdef VBOX_STRICT
5253	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5254	{
5255	switch (idxField)
5256	{
5257	case VMX_VMCS_GUEST_RIP:
5258	case VMX_VMCS_GUEST_RSP:
5259	case VMX_VMCS_GUEST_SYSENTER_EIP:
5260	case VMX_VMCS_GUEST_SYSENTER_ESP:
5261	case VMX_VMCS_GUEST_GDTR_BASE:
5262	case VMX_VMCS_GUEST_IDTR_BASE:
5263	case VMX_VMCS_GUEST_CS_BASE:
5264	case VMX_VMCS_GUEST_DS_BASE:
5265	case VMX_VMCS_GUEST_ES_BASE:
5266	case VMX_VMCS_GUEST_FS_BASE:
5267	case VMX_VMCS_GUEST_GS_BASE:
5268	case VMX_VMCS_GUEST_SS_BASE:
5269	case VMX_VMCS_GUEST_LDTR_BASE:
5270	case VMX_VMCS_GUEST_TR_BASE:
5271	case VMX_VMCS_GUEST_CR3:
5272	return true;
5273	}
5274	return false;
5275	}
5276
5277	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5278	{
5279	switch (idxField)
5280	{
5281	/* Read-only fields. */
5282	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5283	return true;
5284	}
5285	/* Remaining readable fields should also be writable. */
5286	return hmR0VmxIsValidWriteField(idxField);
5287	}
5288	#endif /* VBOX_STRICT */
5289
5290
5291	/**
5292	* Executes the specified handler in 64-bit mode.
5293	*
5294	* @returns VBox status code (no informational status codes).
5295	* @param pVM The cross context VM structure.
5296	* @param pVCpu The cross context virtual CPU structure.
5297	* @param pCtx Pointer to the guest CPU context.
5298	* @param enmOp The operation to perform.
5299	* @param cParams Number of parameters.
5300	* @param paParam Array of 32-bit parameters.
5301	*/
5302	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5303	uint32_t cParams, uint32_t *paParam)
5304	{
5305	NOREF(pCtx);
5306
5307	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5308	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5309	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5310	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5311
5312	#ifdef VBOX_STRICT
5313	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5314	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5315
5316	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5317	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5318	#endif
5319
5320	/* Disable interrupts. */
5321	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5322
5323	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5324	RTCPUID idHostCpu = RTMpCpuId();
5325	CPUMR0SetLApic(pVCpu, idHostCpu);
5326	#endif
5327
5328	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5329	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5330
5331	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5332	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5333	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5334
5335	/* Leave VMX Root Mode. */
5336	VMXDisable();
5337
5338	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5339
5340	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5341	CPUMSetHyperEIP(pVCpu, enmOp);
5342	for (int i = (int)cParams - 1; i >= 0; i--)
5343	CPUMPushHyper(pVCpu, paParam[i]);
5344
5345	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5346
5347	/* Call the switcher. */
5348	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5349	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5350
5351	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5352	/* Make sure the VMX instructions don't cause #UD faults. */
5353	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5354
5355	/* Re-enter VMX Root Mode */
5356	int rc2 = VMXEnable(HCPhysCpuPage);
5357	if (RT_FAILURE(rc2))
5358	{
5359	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5360	ASMSetFlags(fOldEFlags);
5361	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5362	return rc2;
5363	}
5364
5365	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5366	AssertRC(rc2);
5367	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5368	Assert(!(ASMGetFlags() & X86_EFL_IF));
5369	ASMSetFlags(fOldEFlags);
5370	return rc;
5371	}
5372
5373
5374	/**
5375	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5376	* supporting 64-bit guests.
5377	*
5378	* @returns VBox status code.
5379	* @param fResume Whether to VMLAUNCH or VMRESUME.
5380	* @param pCtx Pointer to the guest-CPU context.
5381	* @param pCache Pointer to the VMCS cache.
5382	* @param pVM The cross context VM structure.
5383	* @param pVCpu The cross context virtual CPU structure.
5384	*/
5385	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5386	{
5387	NOREF(fResume);
5388
5389	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5390	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5391
5392	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5393	pCache->uPos = 1;
5394	pCache->interPD = PGMGetInterPaeCR3(pVM);
5395	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5396	#endif
5397
5398	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5399	pCache->TestIn.HCPhysCpuPage = 0;
5400	pCache->TestIn.HCPhysVmcs = 0;
5401	pCache->TestIn.pCache = 0;
5402	pCache->TestOut.HCPhysVmcs = 0;
5403	pCache->TestOut.pCache = 0;
5404	pCache->TestOut.pCtx = 0;
5405	pCache->TestOut.eflags = 0;
5406	#else
5407	NOREF(pCache);
5408	#endif
5409
5410	uint32_t aParam[10];
5411	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5412	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5413	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5414	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5415	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5416	aParam[5] = 0;
5417	aParam[6] = VM_RC_ADDR(pVM, pVM);
5418	aParam[7] = 0;
5419	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5420	aParam[9] = 0;
5421
5422	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5423	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5424	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5425	#endif
5426	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5427
5428	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5429	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5430	Assert(pCtx->dr[4] == 10);
5431	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5432	#endif
5433
5434	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5435	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5436	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5437	pVCpu->hm.s.vmx.HCPhysVmcs));
5438	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5439	pCache->TestOut.HCPhysVmcs));
5440	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5441	pCache->TestOut.pCache));
5442	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5443	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5444	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5445	pCache->TestOut.pCtx));
5446	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5447	#endif
5448	return rc;
5449	}
5450
5451
5452	/**
5453	* Initialize the VMCS-Read cache.
5454	*
5455	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5456	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5457	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5458	* (those that have a 32-bit FULL & HIGH part).
5459	*
5460	* @returns VBox status code.
5461	* @param pVM The cross context VM structure.
5462	* @param pVCpu The cross context virtual CPU structure.
5463	*/
5464	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5465	{
5466	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5467	{ \
5468	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5469	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5470	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5471	++cReadFields; \
5472	}
5473
5474	AssertPtr(pVM);
5475	AssertPtr(pVCpu);
5476	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5477	uint32_t cReadFields = 0;
5478
5479	/*
5480	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5481	* and serve to indicate exceptions to the rules.
5482	*/
5483
5484	/* Guest-natural selector base fields. */
5485	#if 0
5486	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5487	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5488	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5489	#endif
5490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5491	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5492	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5493	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5494	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5495	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5498	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5499	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5502	#if 0
5503	/* Unused natural width guest-state fields. */
5504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5506	#endif
5507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5509
5510	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5511	#if 0
5512	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5513	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5515	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5518	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5519	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5520	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5521	#endif
5522
5523	/* Natural width guest-state fields. */
5524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5525	#if 0
5526	/* Currently unused field. */
5527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5528	#endif
5529
5530	if (pVM->hm.s.fNestedPaging)
5531	{
5532	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5533	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5534	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5535	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5536	}
5537	else
5538	{
5539	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5540	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5541	}
5542
5543	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5544	return VINF_SUCCESS;
5545	}
5546
5547
5548	/**
5549	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5550	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5551	* darwin, running 64-bit guests).
5552	*
5553	* @returns VBox status code.
5554	* @param pVCpu The cross context virtual CPU structure.
5555	* @param idxField The VMCS field encoding.
5556	* @param u64Val 16, 32 or 64-bit value.
5557	*/
5558	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5559	{
5560	int rc;
5561	switch (idxField)
5562	{
5563	/*
5564	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5565	*/
5566	/* 64-bit Control fields. */
5567	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5568	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5569	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5570	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5571	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5572	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5573	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5574	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5575	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5576	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5577	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5578	case VMX_VMCS64_CTRL_EPTP_FULL:
5579	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5580	/* 64-bit Guest-state fields. */
5581	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5582	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5583	case VMX_VMCS64_GUEST_PAT_FULL:
5584	case VMX_VMCS64_GUEST_EFER_FULL:
5585	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5586	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5587	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5588	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5589	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5590	/* 64-bit Host-state fields. */
5591	case VMX_VMCS64_HOST_PAT_FULL:
5592	case VMX_VMCS64_HOST_EFER_FULL:
5593	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5594	{
5595	rc = VMXWriteVmcs32(idxField, u64Val);
5596	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5597	break;
5598	}
5599
5600	/*
5601	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5602	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5603	*/
5604	/* Natural-width Guest-state fields. */
5605	case VMX_VMCS_GUEST_CR3:
5606	case VMX_VMCS_GUEST_ES_BASE:
5607	case VMX_VMCS_GUEST_CS_BASE:
5608	case VMX_VMCS_GUEST_SS_BASE:
5609	case VMX_VMCS_GUEST_DS_BASE:
5610	case VMX_VMCS_GUEST_FS_BASE:
5611	case VMX_VMCS_GUEST_GS_BASE:
5612	case VMX_VMCS_GUEST_LDTR_BASE:
5613	case VMX_VMCS_GUEST_TR_BASE:
5614	case VMX_VMCS_GUEST_GDTR_BASE:
5615	case VMX_VMCS_GUEST_IDTR_BASE:
5616	case VMX_VMCS_GUEST_RSP:
5617	case VMX_VMCS_GUEST_RIP:
5618	case VMX_VMCS_GUEST_SYSENTER_ESP:
5619	case VMX_VMCS_GUEST_SYSENTER_EIP:
5620	{
5621	if (!(u64Val >> 32))
5622	{
5623	/* If this field is 64-bit, VT-x will zero out the top bits. */
5624	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5625	}
5626	else
5627	{
5628	/* Assert that only the 32->64 switcher case should ever come here. */
5629	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5630	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5631	}
5632	break;
5633	}
5634
5635	default:
5636	{
5637	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5638	rc = VERR_INVALID_PARAMETER;
5639	break;
5640	}
5641	}
5642	AssertRCReturn(rc, rc);
5643	return rc;
5644	}
5645
5646
5647	/**
5648	* Queue up a VMWRITE by using the VMCS write cache.
5649	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5650	*
5651	* @param pVCpu The cross context virtual CPU structure.
5652	* @param idxField The VMCS field encoding.
5653	* @param u64Val 16, 32 or 64-bit value.
5654	*/
5655	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5656	{
5657	AssertPtr(pVCpu);
5658	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5659
5660	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5661	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5662
5663	/* Make sure there are no duplicates. */
5664	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5665	{
5666	if (pCache->Write.aField[i] == idxField)
5667	{
5668	pCache->Write.aFieldVal[i] = u64Val;
5669	return VINF_SUCCESS;
5670	}
5671	}
5672
5673	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5674	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5675	pCache->Write.cValidEntries++;
5676	return VINF_SUCCESS;
5677	}
5678	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5679
5680
5681	/**
5682	* Sets up the usage of TSC-offsetting and updates the VMCS.
5683	*
5684	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5685	* VMX preemption timer.
5686	*
5687	* @returns VBox status code.
5688	* @param pVM The cross context VM structure.
5689	* @param pVCpu The cross context virtual CPU structure.
5690	*
5691	* @remarks No-long-jump zone!!!
5692	*/
5693	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5694	{
5695	int rc;
5696	bool fOffsettedTsc;
5697	bool fParavirtTsc;
5698	if (pVM->hm.s.vmx.fUsePreemptTimer)
5699	{
5700	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5701	&fOffsettedTsc, &fParavirtTsc);
5702
5703	/* Make sure the returned values have sane upper and lower boundaries. */
5704	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5705	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5706	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5707	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5708
5709	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5710	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5711	}
5712	else
5713	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5714
5715	/** @todo later optimize this to be done elsewhere and not before every
5716	* VM-entry. */
5717	if (fParavirtTsc)
5718	{
5719	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5720	information before every VM-entry, hence disable it for performance sake. */
5721	#if 0
5722	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5723	AssertRC(rc);
5724	#endif
5725	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5726	}
5727
5728	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5729	{
5730	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5731	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5732
5733	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5734	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5735	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5736	}
5737	else
5738	{
5739	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5740	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5741	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5742	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5743	}
5744	}
5745
5746
5747	/**
5748	* Determines if an exception is a contributory exception.
5749	*
5750	* Contributory exceptions are ones which can cause double-faults unless the
5751	* original exception was a benign exception. Page-fault is intentionally not
5752	* included here as it's a conditional contributory exception.
5753	*
5754	* @returns true if the exception is contributory, false otherwise.
5755	* @param uVector The exception vector.
5756	*/
5757	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5758	{
5759	switch (uVector)
5760	{
5761	case X86_XCPT_GP:
5762	case X86_XCPT_SS:
5763	case X86_XCPT_NP:
5764	case X86_XCPT_TS:
5765	case X86_XCPT_DE:
5766	return true;
5767	default:
5768	break;
5769	}
5770	return false;
5771	}
5772
5773
5774	/**
5775	* Sets an event as a pending event to be injected into the guest.
5776	*
5777	* @param pVCpu The cross context virtual CPU structure.
5778	* @param u32IntInfo The VM-entry interruption-information field.
5779	* @param cbInstr The VM-entry instruction length in bytes (for software
5780	* interrupts, exceptions and privileged software
5781	* exceptions).
5782	* @param u32ErrCode The VM-entry exception error code.
5783	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5784	* page-fault.
5785	*
5786	* @remarks Statistics counter assumes this is a guest event being injected or
5787	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5788	* always incremented.
5789	*/
5790	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5791	RTGCUINTPTR GCPtrFaultAddress)
5792	{
5793	Assert(!pVCpu->hm.s.Event.fPending);
5794	pVCpu->hm.s.Event.fPending = true;
5795	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5796	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5797	pVCpu->hm.s.Event.cbInstr = cbInstr;
5798	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5799	}
5800
5801
5802	/**
5803	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5804	*
5805	* @param pVCpu The cross context virtual CPU structure.
5806	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5807	* out-of-sync. Make sure to update the required fields
5808	* before using them.
5809	*/
5810	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5811	{
5812	NOREF(pMixedCtx);
5813	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5814	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5815	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5816	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5817	}
5818
5819
5820	/**
5821	* Handle a condition that occurred while delivering an event through the guest
5822	* IDT.
5823	*
5824	* @returns Strict VBox status code (i.e. informational status codes too).
5825	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5826	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5827	* to continue execution of the guest which will delivery the \#DF.
5828	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5829	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5830	*
5831	* @param pVCpu The cross context virtual CPU structure.
5832	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5833	* out-of-sync. Make sure to update the required fields
5834	* before using them.
5835	* @param pVmxTransient Pointer to the VMX transient structure.
5836	*
5837	* @remarks No-long-jump zone!!!
5838	*/
5839	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5840	{
5841	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5842
5843	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5844	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5845
5846	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5847	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5848	{
5849	uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5850	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5851
5852	typedef enum
5853	{
5854	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5855	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5856	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5857	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5858	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5859	} VMXREFLECTXCPT;
5860
5861	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5862	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5863	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5864	{
5865	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5866	{
5867	enmReflect = VMXREFLECTXCPT_XCPT;
5868	#ifdef VBOX_STRICT
5869	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5870	&& uExitVector == X86_XCPT_PF)
5871	{
5872	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5873	}
5874	#endif
5875	if ( uExitVector == X86_XCPT_PF
5876	&& uIdtVector == X86_XCPT_PF)
5877	{
5878	pVmxTransient->fVectoringDoublePF = true;
5879	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5880	}
5881	else if ( uExitVector == X86_XCPT_AC
5882	&& uIdtVector == X86_XCPT_AC)
5883	{
5884	enmReflect = VMXREFLECTXCPT_HANG;
5885	Log4(("IDT: Nested #AC - Bad guest\n"));
5886	}
5887	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5888	&& hmR0VmxIsContributoryXcpt(uExitVector)
5889	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5890	\|\| uIdtVector == X86_XCPT_PF))
5891	{
5892	enmReflect = VMXREFLECTXCPT_DF;
5893	}
5894	else if (uIdtVector == X86_XCPT_DF)
5895	enmReflect = VMXREFLECTXCPT_TF;
5896	}
5897	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5898	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5899	{
5900	/*
5901	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5902	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5903	*/
5904	enmReflect = VMXREFLECTXCPT_XCPT;
5905
5906	if (uExitVector == X86_XCPT_PF)
5907	{
5908	pVmxTransient->fVectoringPF = true;
5909	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5910	}
5911	}
5912	}
5913	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5914	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5915	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5916	{
5917	/*
5918	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5919	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5920	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5921	*/
5922	enmReflect = VMXREFLECTXCPT_XCPT;
5923	}
5924
5925	/*
5926	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5927	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5928	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5929	*
5930	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5931	*/
5932	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5933	&& enmReflect == VMXREFLECTXCPT_XCPT
5934	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5935	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5936	{
5937	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5938	}
5939
5940	switch (enmReflect)
5941	{
5942	case VMXREFLECTXCPT_XCPT:
5943	{
5944	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5945	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5946	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5947
5948	uint32_t u32ErrCode = 0;
5949	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5950	{
5951	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5952	AssertRCReturn(rc2, rc2);
5953	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5954	}
5955
5956	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5957	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5958	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5959	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5960	rcStrict = VINF_SUCCESS;
5961	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5962	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5963
5964	break;
5965	}
5966
5967	case VMXREFLECTXCPT_DF:
5968	{
5969	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5970	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5971	rcStrict = VINF_HM_DOUBLE_FAULT;
5972	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5973	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5974
5975	break;
5976	}
5977
5978	case VMXREFLECTXCPT_TF:
5979	{
5980	rcStrict = VINF_EM_RESET;
5981	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5982	uExitVector));
5983	break;
5984	}
5985
5986	case VMXREFLECTXCPT_HANG:
5987	{
5988	rcStrict = VERR_EM_GUEST_CPU_HANG;
5989	break;
5990	}
5991
5992	default:
5993	Assert(rcStrict == VINF_SUCCESS);
5994	break;
5995	}
5996	}
5997	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5998	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5999	&& uExitVector != X86_XCPT_DF
6000	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6001	{
6002	/*
6003	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6004	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6005	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6006	*/
6007	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6008	{
6009	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6010	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6011	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6012	}
6013	}
6014
6015	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6016	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6017	return rcStrict;
6018	}
6019
6020
6021	/**
6022	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6023	*
6024	* @returns VBox status code.
6025	* @param pVCpu The cross context virtual CPU structure.
6026	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6027	* out-of-sync. Make sure to update the required fields
6028	* before using them.
6029	*
6030	* @remarks No-long-jump zone!!!
6031	*/
6032	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6033	{
6034	NOREF(pMixedCtx);
6035
6036	/*
6037	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6038	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6039	*/
6040	VMMRZCallRing3Disable(pVCpu);
6041	HM_DISABLE_PREEMPT();
6042
6043	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6044	{
6045	uint32_t uVal = 0;
6046	uint32_t uShadow = 0;
6047	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6048	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6049	AssertRCReturn(rc, rc);
6050
6051	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6052	CPUMSetGuestCR0(pVCpu, uVal);
6053	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6054	}
6055
6056	HM_RESTORE_PREEMPT();
6057	VMMRZCallRing3Enable(pVCpu);
6058	return VINF_SUCCESS;
6059	}
6060
6061
6062	/**
6063	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6064	*
6065	* @returns VBox status code.
6066	* @param pVCpu The cross context virtual CPU structure.
6067	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6068	* out-of-sync. Make sure to update the required fields
6069	* before using them.
6070	*
6071	* @remarks No-long-jump zone!!!
6072	*/
6073	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6074	{
6075	NOREF(pMixedCtx);
6076
6077	int rc = VINF_SUCCESS;
6078	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6079	{
6080	uint32_t uVal = 0;
6081	uint32_t uShadow = 0;
6082	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6083	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6084	AssertRCReturn(rc, rc);
6085
6086	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6087	CPUMSetGuestCR4(pVCpu, uVal);
6088	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6089	}
6090	return rc;
6091	}
6092
6093
6094	/**
6095	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6096	*
6097	* @returns VBox status code.
6098	* @param pVCpu The cross context virtual CPU structure.
6099	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6100	* out-of-sync. Make sure to update the required fields
6101	* before using them.
6102	*
6103	* @remarks No-long-jump zone!!!
6104	*/
6105	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6106	{
6107	int rc = VINF_SUCCESS;
6108	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6109	{
6110	uint64_t u64Val = 0;
6111	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6112	AssertRCReturn(rc, rc);
6113
6114	pMixedCtx->rip = u64Val;
6115	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6116	}
6117	return rc;
6118	}
6119
6120
6121	/**
6122	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6123	*
6124	* @returns VBox status code.
6125	* @param pVCpu The cross context virtual CPU structure.
6126	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6127	* out-of-sync. Make sure to update the required fields
6128	* before using them.
6129	*
6130	* @remarks No-long-jump zone!!!
6131	*/
6132	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6133	{
6134	int rc = VINF_SUCCESS;
6135	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6136	{
6137	uint64_t u64Val = 0;
6138	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6139	AssertRCReturn(rc, rc);
6140
6141	pMixedCtx->rsp = u64Val;
6142	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6143	}
6144	return rc;
6145	}
6146
6147
6148	/**
6149	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6150	*
6151	* @returns VBox status code.
6152	* @param pVCpu The cross context virtual CPU structure.
6153	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6154	* out-of-sync. Make sure to update the required fields
6155	* before using them.
6156	*
6157	* @remarks No-long-jump zone!!!
6158	*/
6159	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6160	{
6161	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6162	{
6163	uint32_t uVal = 0;
6164	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6165	AssertRCReturn(rc, rc);
6166
6167	pMixedCtx->eflags.u32 = uVal;
6168	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6169	{
6170	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6171	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6172
6173	pMixedCtx->eflags.Bits.u1VM = 0;
6174	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6175	}
6176
6177	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6178	}
6179	return VINF_SUCCESS;
6180	}
6181
6182
6183	/**
6184	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6185	* guest-CPU context.
6186	*/
6187	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6188	{
6189	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6190	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6191	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6192	return rc;
6193	}
6194
6195
6196	/**
6197	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6198	* from the guest-state area in the VMCS.
6199	*
6200	* @param pVCpu The cross context virtual CPU structure.
6201	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6202	* out-of-sync. Make sure to update the required fields
6203	* before using them.
6204	*
6205	* @remarks No-long-jump zone!!!
6206	*/
6207	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6208	{
6209	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6210	{
6211	uint32_t uIntrState = 0;
6212	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6213	AssertRC(rc);
6214
6215	if (!uIntrState)
6216	{
6217	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6218	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6219
6220	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6221	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6222	}
6223	else
6224	{
6225	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6226	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6227	{
6228	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6229	AssertRC(rc);
6230	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6231	AssertRC(rc);
6232
6233	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6234	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6235	}
6236	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6237	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6238
6239	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6240	{
6241	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6242	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6243	}
6244	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6245	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6246	}
6247
6248	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6249	}
6250	}
6251
6252
6253	/**
6254	* Saves the guest's activity state.
6255	*
6256	* @returns VBox status code.
6257	* @param pVCpu The cross context virtual CPU structure.
6258	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6259	* out-of-sync. Make sure to update the required fields
6260	* before using them.
6261	*
6262	* @remarks No-long-jump zone!!!
6263	*/
6264	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6265	{
6266	NOREF(pMixedCtx);
6267	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6268	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6269	return VINF_SUCCESS;
6270	}
6271
6272
6273	/**
6274	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6275	* the current VMCS into the guest-CPU context.
6276	*
6277	* @returns VBox status code.
6278	* @param pVCpu The cross context virtual CPU structure.
6279	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6280	* out-of-sync. Make sure to update the required fields
6281	* before using them.
6282	*
6283	* @remarks No-long-jump zone!!!
6284	*/
6285	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6286	{
6287	int rc = VINF_SUCCESS;
6288	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6289	{
6290	uint32_t u32Val = 0;
6291	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6292	pMixedCtx->SysEnter.cs = u32Val;
6293	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6294	}
6295
6296	uint64_t u64Val = 0;
6297	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6298	{
6299	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6300	pMixedCtx->SysEnter.eip = u64Val;
6301	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6302	}
6303	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6304	{
6305	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6306	pMixedCtx->SysEnter.esp = u64Val;
6307	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6308	}
6309	return rc;
6310	}
6311
6312
6313	/**
6314	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6315	* the CPU back into the guest-CPU context.
6316	*
6317	* @returns VBox status code.
6318	* @param pVCpu The cross context virtual CPU structure.
6319	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6320	* out-of-sync. Make sure to update the required fields
6321	* before using them.
6322	*
6323	* @remarks No-long-jump zone!!!
6324	*/
6325	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6326	{
6327	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6328	VMMRZCallRing3Disable(pVCpu);
6329	HM_DISABLE_PREEMPT();
6330
6331	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6332	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6333	{
6334	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6335	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6336	}
6337
6338	HM_RESTORE_PREEMPT();
6339	VMMRZCallRing3Enable(pVCpu);
6340
6341	return VINF_SUCCESS;
6342	}
6343
6344
6345	/**
6346	* Saves the auto load/store'd guest MSRs from the current VMCS into
6347	* the guest-CPU context.
6348	*
6349	* @returns VBox status code.
6350	* @param pVCpu The cross context virtual CPU structure.
6351	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6352	* out-of-sync. Make sure to update the required fields
6353	* before using them.
6354	*
6355	* @remarks No-long-jump zone!!!
6356	*/
6357	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6358	{
6359	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6360	return VINF_SUCCESS;
6361
6362	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6363	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6364	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6365	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6366	{
6367	switch (pMsr->u32Msr)
6368	{
6369	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6370	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6371	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6372	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6373	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6374	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6375	break;
6376
6377	default:
6378	{
6379	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6380	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6381	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6382	}
6383	}
6384	}
6385
6386	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6387	return VINF_SUCCESS;
6388	}
6389
6390
6391	/**
6392	* Saves the guest control registers from the current VMCS into the guest-CPU
6393	* context.
6394	*
6395	* @returns VBox status code.
6396	* @param pVCpu The cross context virtual CPU structure.
6397	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6398	* out-of-sync. Make sure to update the required fields
6399	* before using them.
6400	*
6401	* @remarks No-long-jump zone!!!
6402	*/
6403	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6404	{
6405	/* Guest CR0. Guest FPU. */
6406	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6407	AssertRCReturn(rc, rc);
6408
6409	/* Guest CR4. */
6410	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6411	AssertRCReturn(rc, rc);
6412
6413	/* Guest CR2 - updated always during the world-switch or in #PF. */
6414	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6415	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6416	{
6417	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6418	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6419
6420	PVM pVM = pVCpu->CTX_SUFF(pVM);
6421	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6422	\|\| ( pVM->hm.s.fNestedPaging
6423	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6424	{
6425	uint64_t u64Val = 0;
6426	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6427	if (pMixedCtx->cr3 != u64Val)
6428	{
6429	CPUMSetGuestCR3(pVCpu, u64Val);
6430	if (VMMRZCallRing3IsEnabled(pVCpu))
6431	{
6432	PGMUpdateCR3(pVCpu, u64Val);
6433	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6434	}
6435	else
6436	{
6437	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6438	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6439	}
6440	}
6441
6442	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6443	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6444	{
6445	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6446	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6447	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6448	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6449	AssertRCReturn(rc, rc);
6450
6451	if (VMMRZCallRing3IsEnabled(pVCpu))
6452	{
6453	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6454	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6455	}
6456	else
6457	{
6458	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6459	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6460	}
6461	}
6462	}
6463
6464	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6465	}
6466
6467	/*
6468	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6469	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6470	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6471	*
6472	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6473	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6474	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6475	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6476	*
6477	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6478	*/
6479	if (VMMRZCallRing3IsEnabled(pVCpu))
6480	{
6481	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6482	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6483
6484	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6485	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6486
6487	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6488	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6489	}
6490
6491	return rc;
6492	}
6493
6494
6495	/**
6496	* Reads a guest segment register from the current VMCS into the guest-CPU
6497	* context.
6498	*
6499	* @returns VBox status code.
6500	* @param pVCpu The cross context virtual CPU structure.
6501	* @param idxSel Index of the selector in the VMCS.
6502	* @param idxLimit Index of the segment limit in the VMCS.
6503	* @param idxBase Index of the segment base in the VMCS.
6504	* @param idxAccess Index of the access rights of the segment in the VMCS.
6505	* @param pSelReg Pointer to the segment selector.
6506	*
6507	* @remarks No-long-jump zone!!!
6508	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6509	* macro as that takes care of whether to read from the VMCS cache or
6510	* not.
6511	*/
6512	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6513	PCPUMSELREG pSelReg)
6514	{
6515	NOREF(pVCpu);
6516
6517	uint32_t u32Val = 0;
6518	int rc = VMXReadVmcs32(idxSel, &u32Val);
6519	AssertRCReturn(rc, rc);
6520	pSelReg->Sel = (uint16_t)u32Val;
6521	pSelReg->ValidSel = (uint16_t)u32Val;
6522	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6523
6524	rc = VMXReadVmcs32(idxLimit, &u32Val);
6525	AssertRCReturn(rc, rc);
6526	pSelReg->u32Limit = u32Val;
6527
6528	uint64_t u64Val = 0;
6529	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6530	AssertRCReturn(rc, rc);
6531	pSelReg->u64Base = u64Val;
6532
6533	rc = VMXReadVmcs32(idxAccess, &u32Val);
6534	AssertRCReturn(rc, rc);
6535	pSelReg->Attr.u = u32Val;
6536
6537	/*
6538	* If VT-x marks the segment as unusable, most other bits remain undefined:
6539	* - For CS the L, D and G bits have meaning.
6540	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6541	* - For the remaining data segments no bits are defined.
6542	*
6543	* The present bit and the unusable bit has been observed to be set at the
6544	* same time (the selector was supposed to be invalid as we started executing
6545	* a V8086 interrupt in ring-0).
6546	*
6547	* What should be important for the rest of the VBox code, is that the P bit is
6548	* cleared. Some of the other VBox code recognizes the unusable bit, but
6549	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6550	* safe side here, we'll strip off P and other bits we don't care about. If
6551	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6552	*
6553	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6554	*/
6555	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6556	{
6557	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6558
6559	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6560	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6561	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6562
6563	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6564	#ifdef DEBUG_bird
6565	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6566	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6567	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6568	#endif
6569	}
6570	return VINF_SUCCESS;
6571	}
6572
6573
6574	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6575	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6576	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6577	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6578	#else
6579	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6580	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6581	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6582	#endif
6583
6584
6585	/**
6586	* Saves the guest segment registers from the current VMCS into the guest-CPU
6587	* context.
6588	*
6589	* @returns VBox status code.
6590	* @param pVCpu The cross context virtual CPU structure.
6591	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6592	* out-of-sync. Make sure to update the required fields
6593	* before using them.
6594	*
6595	* @remarks No-long-jump zone!!!
6596	*/
6597	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6598	{
6599	/* Guest segment registers. */
6600	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6601	{
6602	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6603	AssertRCReturn(rc, rc);
6604
6605	rc = VMXLOCAL_READ_SEG(CS, cs);
6606	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6607	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6608	rc \|= VMXLOCAL_READ_SEG(ES, es);
6609	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6610	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6611	AssertRCReturn(rc, rc);
6612
6613	/* Restore segment attributes for real-on-v86 mode hack. */
6614	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6615	{
6616	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6617	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6618	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6619	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6620	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6621	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6622	}
6623	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6624	}
6625
6626	return VINF_SUCCESS;
6627	}
6628
6629
6630	/**
6631	* Saves the guest descriptor table registers and task register from the current
6632	* VMCS into the guest-CPU context.
6633	*
6634	* @returns VBox status code.
6635	* @param pVCpu The cross context virtual CPU structure.
6636	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6637	* out-of-sync. Make sure to update the required fields
6638	* before using them.
6639	*
6640	* @remarks No-long-jump zone!!!
6641	*/
6642	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6643	{
6644	int rc = VINF_SUCCESS;
6645
6646	/* Guest LDTR. */
6647	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6648	{
6649	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6650	AssertRCReturn(rc, rc);
6651	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6652	}
6653
6654	/* Guest GDTR. */
6655	uint64_t u64Val = 0;
6656	uint32_t u32Val = 0;
6657	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6658	{
6659	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6660	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6661	pMixedCtx->gdtr.pGdt = u64Val;
6662	pMixedCtx->gdtr.cbGdt = u32Val;
6663	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6664	}
6665
6666	/* Guest IDTR. */
6667	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6668	{
6669	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6670	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6671	pMixedCtx->idtr.pIdt = u64Val;
6672	pMixedCtx->idtr.cbIdt = u32Val;
6673	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6674	}
6675
6676	/* Guest TR. */
6677	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6678	{
6679	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6680	AssertRCReturn(rc, rc);
6681
6682	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6683	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6684	{
6685	rc = VMXLOCAL_READ_SEG(TR, tr);
6686	AssertRCReturn(rc, rc);
6687	}
6688	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6689	}
6690	return rc;
6691	}
6692
6693	#undef VMXLOCAL_READ_SEG
6694
6695
6696	/**
6697	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6698	* context.
6699	*
6700	* @returns VBox status code.
6701	* @param pVCpu The cross context virtual CPU structure.
6702	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6703	* out-of-sync. Make sure to update the required fields
6704	* before using them.
6705	*
6706	* @remarks No-long-jump zone!!!
6707	*/
6708	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6709	{
6710	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6711	{
6712	if (!pVCpu->hm.s.fUsingHyperDR7)
6713	{
6714	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6715	uint32_t u32Val;
6716	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6717	pMixedCtx->dr[7] = u32Val;
6718	}
6719
6720	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6721	}
6722	return VINF_SUCCESS;
6723	}
6724
6725
6726	/**
6727	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6728	*
6729	* @returns VBox status code.
6730	* @param pVCpu The cross context virtual CPU structure.
6731	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6732	* out-of-sync. Make sure to update the required fields
6733	* before using them.
6734	*
6735	* @remarks No-long-jump zone!!!
6736	*/
6737	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6738	{
6739	NOREF(pMixedCtx);
6740
6741	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6742	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6743	return VINF_SUCCESS;
6744	}
6745
6746
6747	/**
6748	* Saves the entire guest state from the currently active VMCS into the
6749	* guest-CPU context.
6750	*
6751	* This essentially VMREADs all guest-data.
6752	*
6753	* @returns VBox status code.
6754	* @param pVCpu The cross context virtual CPU structure.
6755	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6756	* out-of-sync. Make sure to update the required fields
6757	* before using them.
6758	*/
6759	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6760	{
6761	Assert(pVCpu);
6762	Assert(pMixedCtx);
6763
6764	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6765	return VINF_SUCCESS;
6766
6767	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6768	again on the ring-3 callback path, there is no real need to. */
6769	if (VMMRZCallRing3IsEnabled(pVCpu))
6770	VMMR0LogFlushDisable(pVCpu);
6771	else
6772	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6773	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6774
6775	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6776	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6777
6778	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6779	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6780
6781	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6782	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6783
6784	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6785	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6786
6787	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6788	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6789
6790	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6791	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6792
6793	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6794	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6795
6796	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6797	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6798
6799	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6800	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6801
6802	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6803	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6804
6805	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6806	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6807	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6808
6809	if (VMMRZCallRing3IsEnabled(pVCpu))
6810	VMMR0LogFlushEnable(pVCpu);
6811
6812	return VINF_SUCCESS;
6813	}
6814
6815
6816	/**
6817	* Saves basic guest registers needed for IEM instruction execution.
6818	*
6819	* @returns VBox status code (OR-able).
6820	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
6821	* @param pMixedCtx Pointer to the CPU context of the guest.
6822	* @param fMemory Whether the instruction being executed operates on
6823	* memory or not. Only CR0 is synced up if clear.
6824	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6825	*/
6826	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6827	{
6828	/*
6829	* We assume all general purpose registers other than RSP are available.
6830	*
6831	* - RIP is a must, as it will be incremented or otherwise changed.
6832	* - RFLAGS are always required to figure the CPL.
6833	* - RSP isn't always required, however it's a GPR, so frequently required.
6834	* - SS and CS are the only segment register needed if IEM doesn't do memory
6835	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6836	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6837	* be required for memory accesses.
6838	*
6839	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
6840	*/
6841	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6842	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6843	if (fNeedRsp)
6844	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6845	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6846	if (!fMemory)
6847	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6848	else
6849	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6850	AssertRCReturn(rc, rc);
6851	return rc;
6852	}
6853
6854
6855	/**
6856	* Ensures that we've got a complete basic guest-context.
6857	*
6858	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
6859	* is for the interpreter.
6860	*
6861	* @returns VBox status code.
6862	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
6863	* @param pMixedCtx Pointer to the guest-CPU context which may have data
6864	* needing to be synced in.
6865	* @thread EMT(pVCpu)
6866	*/
6867	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6868	{
6869	/* Note! Since this is only applicable to VT-x, the implementation is placed
6870	in the VT-x part of the sources instead of the generic stuff. */
6871	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6872	{
6873	/* For now, imply that the caller might change everything too. */
6874	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6875	return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6876	}
6877	return VINF_SUCCESS;
6878	}
6879
6880
6881	/**
6882	* Check per-VM and per-VCPU force flag actions that require us to go back to
6883	* ring-3 for one reason or another.
6884	*
6885	* @returns Strict VBox status code (i.e. informational status codes too)
6886	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6887	* ring-3.
6888	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6889	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6890	* interrupts)
6891	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6892	* all EMTs to be in ring-3.
6893	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6894	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6895	* to the EM loop.
6896	*
6897	* @param pVM The cross context VM structure.
6898	* @param pVCpu The cross context virtual CPU structure.
6899	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6900	* out-of-sync. Make sure to update the required fields
6901	* before using them.
6902	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
6903	*/
6904	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6905	{
6906	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6907
6908	/*
6909	* Anything pending? Should be more likely than not if we're doing a good job.
6910	*/
6911	if ( !fStepping
6912	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6913	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6914	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6915	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6916	return VINF_SUCCESS;
6917
6918	/* We need the control registers now, make sure the guest-CPU context is updated. */
6919	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6920	AssertRCReturn(rc3, rc3);
6921
6922	/* Pending HM CR3 sync. */
6923	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6924	{
6925	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6926	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6927	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6928	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6929	}
6930
6931	/* Pending HM PAE PDPEs. */
6932	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6933	{
6934	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6935	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6936	}
6937
6938	/* Pending PGM C3 sync. */
6939	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6940	{
6941	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6942	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6943	if (rcStrict2 != VINF_SUCCESS)
6944	{
6945	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6946	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6947	return rcStrict2;
6948	}
6949	}
6950
6951	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6952	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6953	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6954	{
6955	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6956	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6957	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6958	return rc2;
6959	}
6960
6961	/* Pending VM request packets, such as hardware interrupts. */
6962	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6963	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6964	{
6965	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6966	return VINF_EM_PENDING_REQUEST;
6967	}
6968
6969	/* Pending PGM pool flushes. */
6970	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6971	{
6972	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6973	return VINF_PGM_POOL_FLUSH_PENDING;
6974	}
6975
6976	/* Pending DMA requests. */
6977	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6978	{
6979	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6980	return VINF_EM_RAW_TO_R3;
6981	}
6982
6983	return VINF_SUCCESS;
6984	}
6985
6986
6987	/**
6988	* Converts any TRPM trap into a pending HM event. This is typically used when
6989	* entering from ring-3 (not longjmp returns).
6990	*
6991	* @param pVCpu The cross context virtual CPU structure.
6992	*/
6993	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6994	{
6995	Assert(TRPMHasTrap(pVCpu));
6996	Assert(!pVCpu->hm.s.Event.fPending);
6997
6998	uint8_t uVector;
6999	TRPMEVENT enmTrpmEvent;
7000	RTGCUINT uErrCode;
7001	RTGCUINTPTR GCPtrFaultAddress;
7002	uint8_t cbInstr;
7003
7004	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7005	AssertRC(rc);
7006
7007	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7008	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7009	if (enmTrpmEvent == TRPM_TRAP)
7010	{
7011	switch (uVector)
7012	{
7013	case X86_XCPT_NMI:
7014	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7015	break;
7016
7017	case X86_XCPT_BP:
7018	case X86_XCPT_OF:
7019	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7020	break;
7021
7022	case X86_XCPT_PF:
7023	case X86_XCPT_DF:
7024	case X86_XCPT_TS:
7025	case X86_XCPT_NP:
7026	case X86_XCPT_SS:
7027	case X86_XCPT_GP:
7028	case X86_XCPT_AC:
7029	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7030	/* no break! */
7031	default:
7032	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7033	break;
7034	}
7035	}
7036	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7037	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7038	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7039	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7040	else
7041	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7042
7043	rc = TRPMResetTrap(pVCpu);
7044	AssertRC(rc);
7045	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7046	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7047
7048	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7049	}
7050
7051
7052	/**
7053	* Converts the pending HM event into a TRPM trap.
7054	*
7055	* @param pVCpu The cross context virtual CPU structure.
7056	*/
7057	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7058	{
7059	Assert(pVCpu->hm.s.Event.fPending);
7060
7061	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7062	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7063	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7064	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7065
7066	/* If a trap was already pending, we did something wrong! */
7067	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7068
7069	TRPMEVENT enmTrapType;
7070	switch (uVectorType)
7071	{
7072	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7073	enmTrapType = TRPM_HARDWARE_INT;
7074	break;
7075
7076	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7077	enmTrapType = TRPM_SOFTWARE_INT;
7078	break;
7079
7080	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7081	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7082	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7083	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7084	enmTrapType = TRPM_TRAP;
7085	break;
7086
7087	default:
7088	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7089	enmTrapType = TRPM_32BIT_HACK;
7090	break;
7091	}
7092
7093	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7094
7095	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7096	AssertRC(rc);
7097
7098	if (fErrorCodeValid)
7099	TRPMSetErrorCode(pVCpu, uErrorCode);
7100
7101	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7102	&& uVector == X86_XCPT_PF)
7103	{
7104	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7105	}
7106	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7107	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7108	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7109	{
7110	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7111	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7112	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7113	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7114	}
7115
7116	/* Clear any pending events from the VMCS. */
7117	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7118	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7119
7120	/* We're now done converting the pending event. */
7121	pVCpu->hm.s.Event.fPending = false;
7122	}
7123
7124
7125	/**
7126	* Does the necessary state syncing before returning to ring-3 for any reason
7127	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7128	*
7129	* @returns VBox status code.
7130	* @param pVCpu The cross context virtual CPU structure.
7131	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7132	* be out-of-sync. Make sure to update the required
7133	* fields before using them.
7134	* @param fSaveGuestState Whether to save the guest state or not.
7135	*
7136	* @remarks No-long-jmp zone!!!
7137	*/
7138	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7139	{
7140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7141	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7142
7143	RTCPUID idCpu = RTMpCpuId();
7144	Log4Func(("HostCpuId=%u\n", idCpu));
7145
7146	/*
7147	* !!! IMPORTANT !!!
7148	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7149	*/
7150
7151	/* Save the guest state if necessary. */
7152	if ( fSaveGuestState
7153	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7154	{
7155	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7156	AssertRCReturn(rc, rc);
7157	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7158	}
7159
7160	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7161	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7162	{
7163	if (fSaveGuestState)
7164	{
7165	/* We shouldn't reload CR0 without saving it first. */
7166	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7167	AssertRCReturn(rc, rc);
7168	}
7169	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7170	}
7171
7172	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7173	#ifdef VBOX_STRICT
7174	if (CPUMIsHyperDebugStateActive(pVCpu))
7175	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7176	#endif
7177	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7178	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7179	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7180	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7181
7182	#if HC_ARCH_BITS == 64
7183	/* Restore host-state bits that VT-x only restores partially. */
7184	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7185	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7186	{
7187	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7188	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7189	}
7190	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7191	#endif
7192
7193	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7194	if (pVCpu->hm.s.vmx.fLazyMsrs)
7195	{
7196	/* We shouldn't reload the guest MSRs without saving it first. */
7197	if (!fSaveGuestState)
7198	{
7199	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7200	AssertRCReturn(rc, rc);
7201	}
7202	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7203	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7204	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7205	}
7206
7207	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7208	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7209
7210	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7211	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7212	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7213	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7214	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7215	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7216	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7217	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7218
7219	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7220
7221	/** @todo This partially defeats the purpose of having preemption hooks.
7222	* The problem is, deregistering the hooks should be moved to a place that
7223	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7224	* context.
7225	*/
7226	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7227	{
7228	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7229	AssertRCReturn(rc, rc);
7230
7231	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7232	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7233	}
7234	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7235	NOREF(idCpu);
7236
7237	return VINF_SUCCESS;
7238	}
7239
7240
7241	/**
7242	* Leaves the VT-x session.
7243	*
7244	* @returns VBox status code.
7245	* @param pVCpu The cross context virtual CPU structure.
7246	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7247	* out-of-sync. Make sure to update the required fields
7248	* before using them.
7249	*
7250	* @remarks No-long-jmp zone!!!
7251	*/
7252	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7253	{
7254	HM_DISABLE_PREEMPT();
7255	HMVMX_ASSERT_CPU_SAFE();
7256	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7257	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7258
7259	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7260	and done this from the VMXR0ThreadCtxCallback(). */
7261	if (!pVCpu->hm.s.fLeaveDone)
7262	{
7263	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7264	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7265	pVCpu->hm.s.fLeaveDone = true;
7266	}
7267	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7268
7269	/*
7270	* !!! IMPORTANT !!!
7271	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7272	*/
7273
7274	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7275	/** @todo Deregistering here means we need to VMCLEAR always
7276	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7277	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7278	VMMR0ThreadCtxHookDisable(pVCpu);
7279
7280	/* Leave HM context. This takes care of local init (term). */
7281	int rc = HMR0LeaveCpu(pVCpu);
7282
7283	HM_RESTORE_PREEMPT();
7284	return rc;
7285	}
7286
7287
7288	/**
7289	* Does the necessary state syncing before doing a longjmp to ring-3.
7290	*
7291	* @returns VBox status code.
7292	* @param pVCpu The cross context virtual CPU structure.
7293	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7294	* out-of-sync. Make sure to update the required fields
7295	* before using them.
7296	*
7297	* @remarks No-long-jmp zone!!!
7298	*/
7299	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7300	{
7301	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7302	}
7303
7304
7305	/**
7306	* Take necessary actions before going back to ring-3.
7307	*
7308	* An action requires us to go back to ring-3. This function does the necessary
7309	* steps before we can safely return to ring-3. This is not the same as longjmps
7310	* to ring-3, this is voluntary and prepares the guest so it may continue
7311	* executing outside HM (recompiler/IEM).
7312	*
7313	* @returns VBox status code.
7314	* @param pVM The cross context VM structure.
7315	* @param pVCpu The cross context virtual CPU structure.
7316	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7317	* out-of-sync. Make sure to update the required fields
7318	* before using them.
7319	* @param rcExit The reason for exiting to ring-3. Can be
7320	* VINF_VMM_UNKNOWN_RING3_CALL.
7321	*/
7322	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7323	{
7324	Assert(pVM);
7325	Assert(pVCpu);
7326	Assert(pMixedCtx);
7327	HMVMX_ASSERT_PREEMPT_SAFE();
7328
7329	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7330	{
7331	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7332	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7333	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7334	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7335	}
7336
7337	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7338	VMMRZCallRing3Disable(pVCpu);
7339	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7340
7341	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7342	if (pVCpu->hm.s.Event.fPending)
7343	{
7344	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7345	Assert(!pVCpu->hm.s.Event.fPending);
7346	}
7347
7348	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7349	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7350
7351	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7352	and if we're injecting an event we should have a TRPM trap pending. */
7353	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7354	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7355	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7356	#endif
7357
7358	/* Save guest state and restore host state bits. */
7359	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7360	AssertRCReturn(rc, rc);
7361	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7362	/* Thread-context hooks are unregistered at this point!!! */
7363
7364	/* Sync recompiler state. */
7365	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7366	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7367	\| CPUM_CHANGED_LDTR
7368	\| CPUM_CHANGED_GDTR
7369	\| CPUM_CHANGED_IDTR
7370	\| CPUM_CHANGED_TR
7371	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7372	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7373	if ( pVM->hm.s.fNestedPaging
7374	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7375	{
7376	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7377	}
7378
7379	Assert(!pVCpu->hm.s.fClearTrapFlag);
7380
7381	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7382	if (rcExit != VINF_EM_RAW_INTERRUPT)
7383	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7384
7385	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7386
7387	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7388	VMMRZCallRing3RemoveNotification(pVCpu);
7389	VMMRZCallRing3Enable(pVCpu);
7390
7391	return rc;
7392	}
7393
7394
7395	/**
7396	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7397	* longjump to ring-3 and possibly get preempted.
7398	*
7399	* @returns VBox status code.
7400	* @param pVCpu The cross context virtual CPU structure.
7401	* @param enmOperation The operation causing the ring-3 longjump.
7402	* @param pvUser Opaque pointer to the guest-CPU context. The data
7403	* may be out-of-sync. Make sure to update the required
7404	* fields before using them.
7405	*/
7406	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7407	{
7408	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7409	{
7410	/*
7411	* !!! IMPORTANT !!!
7412	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7413	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7414	*/
7415	VMMRZCallRing3RemoveNotification(pVCpu);
7416	VMMRZCallRing3Disable(pVCpu);
7417	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7418	RTThreadPreemptDisable(&PreemptState);
7419
7420	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7421	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7422
7423	#if HC_ARCH_BITS == 64
7424	/* Restore host-state bits that VT-x only restores partially. */
7425	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7426	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7427	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7428	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7429	#endif
7430	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7431	if (pVCpu->hm.s.vmx.fLazyMsrs)
7432	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7433
7434	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7435	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7436	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7437	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7438	{
7439	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7440	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7441	}
7442
7443	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7444	VMMR0ThreadCtxHookDisable(pVCpu);
7445	HMR0LeaveCpu(pVCpu);
7446	RTThreadPreemptRestore(&PreemptState);
7447	return VINF_SUCCESS;
7448	}
7449
7450	Assert(pVCpu);
7451	Assert(pvUser);
7452	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7453	HMVMX_ASSERT_PREEMPT_SAFE();
7454
7455	VMMRZCallRing3Disable(pVCpu);
7456	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7457
7458	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7459	enmOperation));
7460
7461	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7462	AssertRCReturn(rc, rc);
7463
7464	VMMRZCallRing3Enable(pVCpu);
7465	return VINF_SUCCESS;
7466	}
7467
7468
7469	/**
7470	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7471	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7472	*
7473	* @param pVCpu The cross context virtual CPU structure.
7474	*/
7475	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7476	{
7477	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7478	{
7479	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7480	{
7481	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7482	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7483	AssertRC(rc);
7484	Log4(("Setup interrupt-window exiting\n"));
7485	}
7486	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7487	}
7488
7489
7490	/**
7491	* Clears the interrupt-window exiting control in the VMCS.
7492	*
7493	* @param pVCpu The cross context virtual CPU structure.
7494	*/
7495	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7496	{
7497	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7498	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7499	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7500	AssertRC(rc);
7501	Log4(("Cleared interrupt-window exiting\n"));
7502	}
7503
7504
7505	/**
7506	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7507	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7508	*
7509	* @param pVCpu The cross context virtual CPU structure.
7510	*/
7511	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7512	{
7513	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7514	{
7515	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7516	{
7517	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7518	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7519	AssertRC(rc);
7520	Log4(("Setup NMI-window exiting\n"));
7521	}
7522	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7523	}
7524
7525
7526	/**
7527	* Clears the NMI-window exiting control in the VMCS.
7528	*
7529	* @param pVCpu The cross context virtual CPU structure.
7530	*/
7531	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7532	{
7533	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7534	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7535	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7536	AssertRC(rc);
7537	Log4(("Cleared NMI-window exiting\n"));
7538	}
7539
7540
7541	/**
7542	* Evaluates the event to be delivered to the guest and sets it as the pending
7543	* event.
7544	*
7545	* @returns The VT-x guest-interruptibility state.
7546	* @param pVCpu The cross context virtual CPU structure.
7547	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7548	* out-of-sync. Make sure to update the required fields
7549	* before using them.
7550	*/
7551	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7552	{
7553	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7554	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7555	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7556	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7557	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7558
7559	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7560	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7561	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7562	Assert(!TRPMHasTrap(pVCpu));
7563
7564	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7565	APICUpdatePendingInterrupts(pVCpu);
7566
7567	/*
7568	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7569	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7570	*/
7571	/** @todo SMI. SMIs take priority over NMIs. */
7572	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7573	{
7574	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7575	if ( !pVCpu->hm.s.Event.fPending
7576	&& !fBlockNmi
7577	&& !fBlockSti
7578	&& !fBlockMovSS)
7579	{
7580	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7581	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7582	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7583
7584	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7585	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7586	}
7587	else
7588	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7589	}
7590	/*
7591	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7592	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7593	*/
7594	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7595	&& !pVCpu->hm.s.fSingleInstruction)
7596	{
7597	Assert(!DBGFIsStepping(pVCpu));
7598	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7599	AssertRC(rc);
7600	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7601	if ( !pVCpu->hm.s.Event.fPending
7602	&& !fBlockInt
7603	&& !fBlockSti
7604	&& !fBlockMovSS)
7605	{
7606	uint8_t u8Interrupt;
7607	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7608	if (RT_SUCCESS(rc))
7609	{
7610	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7611	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7612	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7613
7614	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7615	}
7616	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7617	{
7618	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7619	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7620	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7621
7622	/*
7623	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7624	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7625	* need to re-set this force-flag here.
7626	*/
7627	}
7628	else
7629	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7630	}
7631	else
7632	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7633	}
7634
7635	return uIntrState;
7636	}
7637
7638
7639	/**
7640	* Sets a pending-debug exception to be delivered to the guest if the guest is
7641	* single-stepping in the VMCS.
7642	*
7643	* @param pVCpu The cross context virtual CPU structure.
7644	*/
7645	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7646	{
7647	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7648	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7649	AssertRC(rc);
7650	}
7651
7652
7653	/**
7654	* Injects any pending events into the guest if the guest is in a state to
7655	* receive them.
7656	*
7657	* @returns Strict VBox status code (i.e. informational status codes too).
7658	* @param pVCpu The cross context virtual CPU structure.
7659	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7660	* out-of-sync. Make sure to update the required fields
7661	* before using them.
7662	* @param uIntrState The VT-x guest-interruptibility state.
7663	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7664	* return VINF_EM_DBG_STEPPED if the event was
7665	* dispatched directly.
7666	*/
7667	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7668	{
7669	HMVMX_ASSERT_PREEMPT_SAFE();
7670	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7671
7672	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7673	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7674
7675	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7676	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7677	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7678	Assert(!TRPMHasTrap(pVCpu));
7679
7680	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7681	if (pVCpu->hm.s.Event.fPending)
7682	{
7683	/*
7684	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7685	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7686	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7687	*
7688	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7689	*/
7690	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7691	#ifdef VBOX_STRICT
7692	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7693	{
7694	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7695	Assert(!fBlockInt);
7696	Assert(!fBlockSti);
7697	Assert(!fBlockMovSS);
7698	}
7699	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7700	{
7701	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7702	Assert(!fBlockSti);
7703	Assert(!fBlockMovSS);
7704	Assert(!fBlockNmi);
7705	}
7706	#endif
7707	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7708	(uint8_t)uIntType));
7709	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7710	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7711	fStepping, &uIntrState);
7712	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7713
7714	/* Update the interruptibility-state as it could have been changed by
7715	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7716	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7717	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7718
7719	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7720	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7721	else
7722	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7723	}
7724
7725	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7726	if ( fBlockSti
7727	\|\| fBlockMovSS)
7728	{
7729	if (!pVCpu->hm.s.fSingleInstruction)
7730	{
7731	/*
7732	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7733	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7734	* See Intel spec. 27.3.4 "Saving Non-Register State".
7735	*/
7736	Assert(!DBGFIsStepping(pVCpu));
7737	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7738	AssertRCReturn(rc2, rc2);
7739	if (pMixedCtx->eflags.Bits.u1TF)
7740	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7741	}
7742	else if (pMixedCtx->eflags.Bits.u1TF)
7743	{
7744	/*
7745	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7746	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7747	*/
7748	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7749	uIntrState = 0;
7750	}
7751	}
7752
7753	/*
7754	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7755	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7756	*/
7757	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7758	AssertRC(rc2);
7759
7760	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7761	NOREF(fBlockMovSS); NOREF(fBlockSti);
7762	return rcStrict;
7763	}
7764
7765
7766	/**
7767	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7768	*
7769	* @param pVCpu The cross context virtual CPU structure.
7770	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7771	* out-of-sync. Make sure to update the required fields
7772	* before using them.
7773	*/
7774	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7775	{
7776	NOREF(pMixedCtx);
7777	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7778	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7779	}
7780
7781
7782	/**
7783	* Injects a double-fault (\#DF) exception into the VM.
7784	*
7785	* @returns Strict VBox status code (i.e. informational status codes too).
7786	* @param pVCpu The cross context virtual CPU structure.
7787	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7788	* out-of-sync. Make sure to update the required fields
7789	* before using them.
7790	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7791	* and should return VINF_EM_DBG_STEPPED if the event
7792	* is injected directly (register modified by us, not
7793	* by hardware on VM-entry).
7794	* @param puIntrState Pointer to the current guest interruptibility-state.
7795	* This interruptibility-state will be updated if
7796	* necessary. This cannot not be NULL.
7797	*/
7798	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7799	{
7800	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7801	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7802	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7803	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7804	fStepping, puIntrState);
7805	}
7806
7807
7808	/**
7809	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7810	*
7811	* @param pVCpu The cross context virtual CPU structure.
7812	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7813	* out-of-sync. Make sure to update the required fields
7814	* before using them.
7815	*/
7816	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7817	{
7818	NOREF(pMixedCtx);
7819	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7820	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7821	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7822	}
7823
7824
7825	/**
7826	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7827	*
7828	* @param pVCpu The cross context virtual CPU structure.
7829	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7830	* out-of-sync. Make sure to update the required fields
7831	* before using them.
7832	* @param cbInstr The value of RIP that is to be pushed on the guest
7833	* stack.
7834	*/
7835	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7836	{
7837	NOREF(pMixedCtx);
7838	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7839	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7840	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7841	}
7842
7843
7844	/**
7845	* Injects a general-protection (\#GP) fault into the VM.
7846	*
7847	* @returns Strict VBox status code (i.e. informational status codes too).
7848	* @param pVCpu The cross context virtual CPU structure.
7849	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7850	* out-of-sync. Make sure to update the required fields
7851	* before using them.
7852	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7853	* mode, i.e. in real-mode it's not valid).
7854	* @param u32ErrorCode The error code associated with the \#GP.
7855	* @param fStepping Whether we're running in
7856	* hmR0VmxRunGuestCodeStep() and should return
7857	* VINF_EM_DBG_STEPPED if the event is injected
7858	* directly (register modified by us, not by
7859	* hardware on VM-entry).
7860	* @param puIntrState Pointer to the current guest interruptibility-state.
7861	* This interruptibility-state will be updated if
7862	* necessary. This cannot not be NULL.
7863	*/
7864	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7865	bool fStepping, uint32_t *puIntrState)
7866	{
7867	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7868	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7869	if (fErrorCodeValid)
7870	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7871	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7872	fStepping, puIntrState);
7873	}
7874
7875
7876	#if 0 /* unused */
7877	/**
7878	* Sets a general-protection (\#GP) exception as pending-for-injection into the
7879	* VM.
7880	*
7881	* @param pVCpu The cross context virtual CPU structure.
7882	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7883	* out-of-sync. Make sure to update the required fields
7884	* before using them.
7885	* @param u32ErrorCode The error code associated with the \#GP.
7886	*/
7887	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7888	{
7889	NOREF(pMixedCtx);
7890	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7891	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7892	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7893	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
7894	}
7895	#endif /* unused */
7896
7897
7898	/**
7899	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7900	*
7901	* @param pVCpu The cross context virtual CPU structure.
7902	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7903	* out-of-sync. Make sure to update the required fields
7904	* before using them.
7905	* @param uVector The software interrupt vector number.
7906	* @param cbInstr The value of RIP that is to be pushed on the guest
7907	* stack.
7908	*/
7909	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7910	{
7911	NOREF(pMixedCtx);
7912	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7913	if ( uVector == X86_XCPT_BP
7914	\|\| uVector == X86_XCPT_OF)
7915	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7916	else
7917	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7918	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7919	}
7920
7921
7922	/**
7923	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7924	* stack.
7925	*
7926	* @returns Strict VBox status code (i.e. informational status codes too).
7927	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7928	* @param pVM The cross context VM structure.
7929	* @param pMixedCtx Pointer to the guest-CPU context.
7930	* @param uValue The value to push to the guest stack.
7931	*/
7932	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7933	{
7934	/*
7935	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7936	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7937	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7938	*/
7939	if (pMixedCtx->sp == 1)
7940	return VINF_EM_RESET;
7941	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7942	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7943	AssertRC(rc);
7944	return rc;
7945	}
7946
7947
7948	/**
7949	* Injects an event into the guest upon VM-entry by updating the relevant fields
7950	* in the VM-entry area in the VMCS.
7951	*
7952	* @returns Strict VBox status code (i.e. informational status codes too).
7953	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7954	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7955	*
7956	* @param pVCpu The cross context virtual CPU structure.
7957	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7958	* be out-of-sync. Make sure to update the required
7959	* fields before using them.
7960	* @param u64IntInfo The VM-entry interruption-information field.
7961	* @param cbInstr The VM-entry instruction length in bytes (for
7962	* software interrupts, exceptions and privileged
7963	* software exceptions).
7964	* @param u32ErrCode The VM-entry exception error code.
7965	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7966	* @param puIntrState Pointer to the current guest interruptibility-state.
7967	* This interruptibility-state will be updated if
7968	* necessary. This cannot not be NULL.
7969	* @param fStepping Whether we're running in
7970	* hmR0VmxRunGuestCodeStep() and should return
7971	* VINF_EM_DBG_STEPPED if the event is injected
7972	* directly (register modified by us, not by
7973	* hardware on VM-entry).
7974	*
7975	* @remarks Requires CR0!
7976	* @remarks No-long-jump zone!!!
7977	*/
7978	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7979	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7980	uint32_t *puIntrState)
7981	{
7982	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7983	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7984	Assert(puIntrState);
7985	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7986
7987	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7988	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7989
7990	#ifdef VBOX_STRICT
7991	/* Validate the error-code-valid bit for hardware exceptions. */
7992	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7993	{
7994	switch (uVector)
7995	{
7996	case X86_XCPT_PF:
7997	case X86_XCPT_DF:
7998	case X86_XCPT_TS:
7999	case X86_XCPT_NP:
8000	case X86_XCPT_SS:
8001	case X86_XCPT_GP:
8002	case X86_XCPT_AC:
8003	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8004	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8005	/* fallthru */
8006	default:
8007	break;
8008	}
8009	}
8010	#endif
8011
8012	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8013	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8014	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8015
8016	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8017
8018	/* We require CR0 to check if the guest is in real-mode. */
8019	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8020	AssertRCReturn(rc, rc);
8021
8022	/*
8023	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8024	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8025	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8026	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8027	*/
8028	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8029	{
8030	PVM pVM = pVCpu->CTX_SUFF(pVM);
8031	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8032	{
8033	Assert(PDMVmmDevHeapIsEnabled(pVM));
8034	Assert(pVM->hm.s.vmx.pRealModeTSS);
8035
8036	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8037	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8038	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8039	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8040	AssertRCReturn(rc, rc);
8041	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8042
8043	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8044	size_t const cbIdtEntry = sizeof(X86IDTR16);
8045	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8046	{
8047	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8048	if (uVector == X86_XCPT_DF)
8049	return VINF_EM_RESET;
8050
8051	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8052	if (uVector == X86_XCPT_GP)
8053	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8054
8055	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8056	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8057	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8058	fStepping, puIntrState);
8059	}
8060
8061	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8062	uint16_t uGuestIp = pMixedCtx->ip;
8063	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8064	{
8065	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8066	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8067	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8068	}
8069	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8070	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8071
8072	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8073	X86IDTR16 IdtEntry;
8074	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8075	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8076	AssertRCReturn(rc, rc);
8077
8078	/* Construct the stack frame for the interrupt/exception handler. */
8079	VBOXSTRICTRC rcStrict;
8080	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8081	if (rcStrict == VINF_SUCCESS)
8082	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8083	if (rcStrict == VINF_SUCCESS)
8084	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8085
8086	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8087	if (rcStrict == VINF_SUCCESS)
8088	{
8089	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8090	pMixedCtx->rip = IdtEntry.offSel;
8091	pMixedCtx->cs.Sel = IdtEntry.uSel;
8092	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8093	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8094	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8095	&& uVector == X86_XCPT_PF)
8096	pMixedCtx->cr2 = GCPtrFaultAddress;
8097
8098	/* If any other guest-state bits are changed here, make sure to update
8099	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8100	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8101	\| HM_CHANGED_GUEST_RIP
8102	\| HM_CHANGED_GUEST_RFLAGS
8103	\| HM_CHANGED_GUEST_RSP);
8104
8105	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8106	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8107	{
8108	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8109	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8110	Log4(("Clearing inhibition due to STI.\n"));
8111	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8112	}
8113	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8114	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8115
8116	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8117	it, if we are returning to ring-3 before executing guest code. */
8118	pVCpu->hm.s.Event.fPending = false;
8119
8120	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8121	if (fStepping)
8122	rcStrict = VINF_EM_DBG_STEPPED;
8123	}
8124	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8125	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8126	return rcStrict;
8127	}
8128
8129	/*
8130	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8131	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8132	*/
8133	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8134	}
8135
8136	/* Validate. */
8137	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8138	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8139	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8140
8141	/* Inject. */
8142	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8143	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8144	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8145	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8146
8147	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8148	&& uVector == X86_XCPT_PF)
8149	pMixedCtx->cr2 = GCPtrFaultAddress;
8150
8151	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8152	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8153
8154	AssertRCReturn(rc, rc);
8155	return VINF_SUCCESS;
8156	}
8157
8158
8159	/**
8160	* Clears the interrupt-window exiting control in the VMCS and if necessary
8161	* clears the current event in the VMCS as well.
8162	*
8163	* @returns VBox status code.
8164	* @param pVCpu The cross context virtual CPU structure.
8165	*
8166	* @remarks Use this function only to clear events that have not yet been
8167	* delivered to the guest but are injected in the VMCS!
8168	* @remarks No-long-jump zone!!!
8169	*/
8170	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8171	{
8172	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8173
8174	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8175	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8176
8177	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8178	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8179	}
8180
8181
8182	/**
8183	* Enters the VT-x session.
8184	*
8185	* @returns VBox status code.
8186	* @param pVM The cross context VM structure.
8187	* @param pVCpu The cross context virtual CPU structure.
8188	* @param pCpu Pointer to the CPU info struct.
8189	*/
8190	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8191	{
8192	AssertPtr(pVM);
8193	AssertPtr(pVCpu);
8194	Assert(pVM->hm.s.vmx.fSupported);
8195	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8196	NOREF(pCpu); NOREF(pVM);
8197
8198	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8199	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8200
8201	#ifdef VBOX_STRICT
8202	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8203	RTCCUINTREG uHostCR4 = ASMGetCR4();
8204	if (!(uHostCR4 & X86_CR4_VMXE))
8205	{
8206	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8207	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8208	}
8209	#endif
8210
8211	/*
8212	* Load the VCPU's VMCS as the current (and active) one.
8213	*/
8214	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8215	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8216	if (RT_FAILURE(rc))
8217	return rc;
8218
8219	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8220	pVCpu->hm.s.fLeaveDone = false;
8221	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8222
8223	return VINF_SUCCESS;
8224	}
8225
8226
8227	/**
8228	* The thread-context callback (only on platforms which support it).
8229	*
8230	* @param enmEvent The thread-context event.
8231	* @param pVCpu The cross context virtual CPU structure.
8232	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8233	* @thread EMT(pVCpu)
8234	*/
8235	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8236	{
8237	NOREF(fGlobalInit);
8238
8239	switch (enmEvent)
8240	{
8241	case RTTHREADCTXEVENT_OUT:
8242	{
8243	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8244	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8245	VMCPU_ASSERT_EMT(pVCpu);
8246
8247	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8248
8249	/* No longjmps (logger flushes, locks) in this fragile context. */
8250	VMMRZCallRing3Disable(pVCpu);
8251	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8252
8253	/*
8254	* Restore host-state (FPU, debug etc.)
8255	*/
8256	if (!pVCpu->hm.s.fLeaveDone)
8257	{
8258	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8259	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8260	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8261	pVCpu->hm.s.fLeaveDone = true;
8262	}
8263
8264	/* Leave HM context, takes care of local init (term). */
8265	int rc = HMR0LeaveCpu(pVCpu);
8266	AssertRC(rc); NOREF(rc);
8267
8268	/* Restore longjmp state. */
8269	VMMRZCallRing3Enable(pVCpu);
8270	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8271	break;
8272	}
8273
8274	case RTTHREADCTXEVENT_IN:
8275	{
8276	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8277	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8278	VMCPU_ASSERT_EMT(pVCpu);
8279
8280	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8281	VMMRZCallRing3Disable(pVCpu);
8282	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8283
8284	/* Initialize the bare minimum state required for HM. This takes care of
8285	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8286	int rc = HMR0EnterCpu(pVCpu);
8287	AssertRC(rc);
8288	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8289
8290	/* Load the active VMCS as the current one. */
8291	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8292	{
8293	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8294	AssertRC(rc); NOREF(rc);
8295	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8296	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8297	}
8298	pVCpu->hm.s.fLeaveDone = false;
8299
8300	/* Restore longjmp state. */
8301	VMMRZCallRing3Enable(pVCpu);
8302	break;
8303	}
8304
8305	default:
8306	break;
8307	}
8308	}
8309
8310
8311	/**
8312	* Saves the host state in the VMCS host-state.
8313	* Sets up the VM-exit MSR-load area.
8314	*
8315	* The CPU state will be loaded from these fields on every successful VM-exit.
8316	*
8317	* @returns VBox status code.
8318	* @param pVM The cross context VM structure.
8319	* @param pVCpu The cross context virtual CPU structure.
8320	*
8321	* @remarks No-long-jump zone!!!
8322	*/
8323	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8324	{
8325	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8326
8327	int rc = VINF_SUCCESS;
8328	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8329	{
8330	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8331	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8332
8333	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8335
8336	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8338
8339	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8340	}
8341	return rc;
8342	}
8343
8344
8345	/**
8346	* Saves the host state in the VMCS host-state.
8347	*
8348	* @returns VBox status code.
8349	* @param pVM The cross context VM structure.
8350	* @param pVCpu The cross context virtual CPU structure.
8351	*
8352	* @remarks No-long-jump zone!!!
8353	*/
8354	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8355	{
8356	AssertPtr(pVM);
8357	AssertPtr(pVCpu);
8358
8359	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8360
8361	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8362	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8363	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8364	return hmR0VmxSaveHostState(pVM, pVCpu);
8365	}
8366
8367
8368	/**
8369	* Loads the guest state into the VMCS guest-state area.
8370	*
8371	* The will typically be done before VM-entry when the guest-CPU state and the
8372	* VMCS state may potentially be out of sync.
8373	*
8374	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8375	* VM-entry controls.
8376	* Sets up the appropriate VMX non-root function to execute guest code based on
8377	* the guest CPU mode.
8378	*
8379	* @returns VBox strict status code.
8380	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8381	* without unrestricted guest access and the VMMDev is not presently
8382	* mapped (e.g. EFI32).
8383	*
8384	* @param pVM The cross context VM structure.
8385	* @param pVCpu The cross context virtual CPU structure.
8386	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8387	* out-of-sync. Make sure to update the required fields
8388	* before using them.
8389	*
8390	* @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8391	* caller disables then again on successfull return. Confusing.)
8392	*/
8393	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8394	{
8395	AssertPtr(pVM);
8396	AssertPtr(pVCpu);
8397	AssertPtr(pMixedCtx);
8398	HMVMX_ASSERT_PREEMPT_SAFE();
8399
8400	VMMRZCallRing3Disable(pVCpu);
8401	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8402
8403	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8404
8405	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8406
8407	/* Determine real-on-v86 mode. */
8408	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8409	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8410	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8411	{
8412	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8413	}
8414
8415	/*
8416	* Load the guest-state into the VMCS.
8417	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8418	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8419	*/
8420	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8421	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8422
8423	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8424	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8425	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8426
8427	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8428	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8429	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8430
8431	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8432	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8433
8434	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8435	if (rcStrict == VINF_SUCCESS)
8436	{ /* likely */ }
8437	else
8438	{
8439	VMMRZCallRing3Enable(pVCpu);
8440	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8441	return rcStrict;
8442	}
8443
8444	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8445	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8446	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8447
8448	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8449	determine we don't have to swap EFER after all. */
8450	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8451	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8452
8453	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8454	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8455
8456	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8457	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8458
8459	/*
8460	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8461	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8462	*/
8463	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8464	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8465
8466	/* Clear any unused and reserved bits. */
8467	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8468
8469	VMMRZCallRing3Enable(pVCpu);
8470
8471	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8472	return rc;
8473	}
8474
8475
8476	/**
8477	* Loads the state shared between the host and guest into the VMCS.
8478	*
8479	* @param pVM The cross context VM structure.
8480	* @param pVCpu The cross context virtual CPU structure.
8481	* @param pCtx Pointer to the guest-CPU context.
8482	*
8483	* @remarks No-long-jump zone!!!
8484	*/
8485	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8486	{
8487	NOREF(pVM);
8488
8489	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8490	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8491
8492	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8493	{
8494	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8495	AssertRC(rc);
8496	}
8497
8498	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8499	{
8500	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8501	AssertRC(rc);
8502
8503	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8504	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8505	{
8506	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8507	AssertRC(rc);
8508	}
8509	}
8510
8511	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8512	{
8513	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8514	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8515	}
8516
8517	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8518	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8519	{
8520	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8521	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8522	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8523	AssertRC(rc);
8524	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8525	}
8526
8527	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8528	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8529	}
8530
8531
8532	/**
8533	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8534	*
8535	* @returns Strict VBox status code (i.e. informational status codes too).
8536	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8537	* without unrestricted guest access and the VMMDev is not presently
8538	* mapped (e.g. EFI32).
8539	*
8540	* @param pVM The cross context VM structure.
8541	* @param pVCpu The cross context virtual CPU structure.
8542	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8543	* out-of-sync. Make sure to update the required fields
8544	* before using them.
8545	*/
8546	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8547	{
8548	HMVMX_ASSERT_PREEMPT_SAFE();
8549
8550	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8551	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8552	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8553	#endif
8554
8555	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8556	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8557	{
8558	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8559	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8560	{ /* likely */}
8561	else
8562	{
8563	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8564	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8565	}
8566	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8567	}
8568	else if (HMCPU_CF_VALUE(pVCpu))
8569	{
8570	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8571	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8572	{ /* likely */}
8573	else
8574	{
8575	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8576	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8577	return rcStrict;
8578	}
8579	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8580	}
8581
8582	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8583	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8584	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8585	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8586	return rcStrict;
8587	}
8588
8589
8590	/**
8591	* Does the preparations before executing guest code in VT-x.
8592	*
8593	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8594	* recompiler/IEM. We must be cautious what we do here regarding committing
8595	* guest-state information into the VMCS assuming we assuredly execute the
8596	* guest in VT-x mode.
8597	*
8598	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8599	* the common-state (TRPM/forceflags), we must undo those changes so that the
8600	* recompiler/IEM can (and should) use them when it resumes guest execution.
8601	* Otherwise such operations must be done when we can no longer exit to ring-3.
8602	*
8603	* @returns Strict VBox status code (i.e. informational status codes too).
8604	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8605	* have been disabled.
8606	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8607	* double-fault into the guest.
8608	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8609	* dispatched directly.
8610	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8611	*
8612	* @param pVM The cross context VM structure.
8613	* @param pVCpu The cross context virtual CPU structure.
8614	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8615	* out-of-sync. Make sure to update the required fields
8616	* before using them.
8617	* @param pVmxTransient Pointer to the VMX transient structure.
8618	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8619	* us ignore some of the reasons for returning to
8620	* ring-3, and return VINF_EM_DBG_STEPPED if event
8621	* dispatching took place.
8622	*/
8623	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8624	{
8625	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8626
8627	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8628	PGMRZDynMapFlushAutoSet(pVCpu);
8629	#endif
8630
8631	/* Check force flag actions that might require us to go back to ring-3. */
8632	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8633	if (rcStrict == VINF_SUCCESS)
8634	{ /* FFs doesn't get set all the time. */ }
8635	else
8636	return rcStrict;
8637
8638	if (TRPMHasTrap(pVCpu))
8639	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8640	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8641
8642	/*
8643	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8644	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8645	*/
8646	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8647	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8648	{ /* likely */ }
8649	else
8650	{
8651	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8652	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8653	return rcStrict;
8654	}
8655
8656	/*
8657	* Load the guest state bits, we can handle longjmps/getting preempted here.
8658	*
8659	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8660	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8661	* Hence, this needs to be done -after- injection of events.
8662	*/
8663	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8664	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8665	{ /* likely */ }
8666	else
8667	return rcStrict;
8668
8669	/*
8670	* No longjmps to ring-3 from this point on!!!
8671	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8672	* This also disables flushing of the R0-logger instance (if any).
8673	*/
8674	VMMRZCallRing3Disable(pVCpu);
8675
8676	/*
8677	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8678	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8679	*
8680	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8681	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8682	*
8683	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8684	* executing guest code.
8685	*/
8686	pVmxTransient->fEFlags = ASMIntDisableFlags();
8687
8688	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8689	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8690	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8691	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8692	{
8693	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8694	{
8695	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8696	pVCpu->hm.s.Event.fPending = false;
8697
8698	return VINF_SUCCESS;
8699	}
8700
8701	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8702	rcStrict = VINF_EM_RAW_INTERRUPT;
8703	}
8704	else
8705	{
8706	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8707	rcStrict = VINF_EM_RAW_TO_R3;
8708	}
8709
8710	ASMSetFlags(pVmxTransient->fEFlags);
8711	VMMRZCallRing3Enable(pVCpu);
8712
8713	return rcStrict;
8714	}
8715
8716
8717	/**
8718	* Prepares to run guest code in VT-x and we've committed to doing so. This
8719	* means there is no backing out to ring-3 or anywhere else at this
8720	* point.
8721	*
8722	* @param pVM The cross context VM structure.
8723	* @param pVCpu The cross context virtual CPU structure.
8724	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8725	* out-of-sync. Make sure to update the required fields
8726	* before using them.
8727	* @param pVmxTransient Pointer to the VMX transient structure.
8728	*
8729	* @remarks Called with preemption disabled.
8730	* @remarks No-long-jump zone!!!
8731	*/
8732	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8733	{
8734	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8735	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8736	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8737
8738	/*
8739	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8740	*/
8741	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8742	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8743
8744	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8745	if (!CPUMIsGuestFPUStateActive(pVCpu))
8746	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8747	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8748	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8749	#endif
8750
8751	if ( pVCpu->hm.s.fPreloadGuestFpu
8752	&& !CPUMIsGuestFPUStateActive(pVCpu))
8753	{
8754	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8755	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8756	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8757	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8758	}
8759
8760	/*
8761	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8762	*/
8763	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8764	&& pVCpu->hm.s.vmx.cMsrs > 0)
8765	{
8766	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8767	}
8768
8769	/*
8770	* Load the host state bits as we may've been preempted (only happens when
8771	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8772	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8773	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
8774	* See @bugref{8432}.
8775	*/
8776	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8777	{
8778	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8779	AssertRC(rc);
8780	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8781	}
8782	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8783
8784	/*
8785	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8786	*/
8787	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8788	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8789	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8790
8791	/* Store status of the shared guest-host state at the time of VM-entry. */
8792	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8793	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8794	{
8795	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8796	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8797	}
8798	else
8799	#endif
8800	{
8801	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8802	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8803	}
8804	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8805
8806	/*
8807	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8808	*/
8809	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8810	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8811
8812	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8813	RTCPUID idCurrentCpu = pCpu->idCpu;
8814	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8815	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8816	{
8817	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8818	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8819	}
8820
8821	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8822	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8823	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8824	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8825
8826	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8827
8828	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8829	to start executing. */
8830
8831	/*
8832	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8833	*/
8834	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8835	{
8836	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8837	{
8838	bool fMsrUpdated;
8839	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8840	AssertRC(rc2);
8841	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8842
8843	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8844	&fMsrUpdated);
8845	AssertRC(rc2);
8846	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8847
8848	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8849	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8850	}
8851	else
8852	{
8853	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8854	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8855	}
8856	}
8857
8858	#ifdef VBOX_STRICT
8859	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8860	hmR0VmxCheckHostEferMsr(pVCpu);
8861	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8862	#endif
8863	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8864	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8865	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8866	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8867	#endif
8868	}
8869
8870
8871	/**
8872	* Performs some essential restoration of state after running guest code in
8873	* VT-x.
8874	*
8875	* @param pVM The cross context VM structure.
8876	* @param pVCpu The cross context virtual CPU structure.
8877	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8878	* out-of-sync. Make sure to update the required fields
8879	* before using them.
8880	* @param pVmxTransient Pointer to the VMX transient structure.
8881	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8882	*
8883	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
8884	*
8885	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8886	* unconditionally when it is safe to do so.
8887	*/
8888	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8889	{
8890	NOREF(pVM);
8891
8892	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8893
8894	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8895	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8896	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8897	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8898	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8899	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8900
8901	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8902	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8903
8904	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8905	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8906	Assert(!ASMIntAreEnabled());
8907	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8908
8909	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8910	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8911	{
8912	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8913	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8914	}
8915	#endif
8916
8917	#if HC_ARCH_BITS == 64
8918	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8919	#endif
8920	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8921	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8922	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8923	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8924	#else
8925	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8926	#endif
8927	#ifdef VBOX_STRICT
8928	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8929	#endif
8930	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8931	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8932
8933	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8934	uint32_t uExitReason;
8935	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8936	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8937	AssertRC(rc);
8938	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8939	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8940
8941	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8942	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8943	{
8944	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8945	pVmxTransient->fVMEntryFailed));
8946	return;
8947	}
8948
8949	/*
8950	* Update the VM-exit history array here even if the VM-entry failed due to:
8951	* - Invalid guest state.
8952	* - MSR loading.
8953	* - Machine-check event.
8954	*
8955	* In any of the above cases we will still have a "valid" VM-exit reason
8956	* despite @a fVMEntryFailed being false.
8957	*
8958	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8959	*/
8960	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8961
8962	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8963	{
8964	/** @todo We can optimize this by only syncing with our force-flags when
8965	* really needed and keeping the VMCS state as it is for most
8966	* VM-exits. */
8967	/* Update the guest interruptibility-state from the VMCS. */
8968	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8969
8970	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8971	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8972	AssertRC(rc);
8973	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8974	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8975	AssertRC(rc);
8976	#endif
8977
8978	/*
8979	* Sync the TPR shadow with our APIC state.
8980	*/
8981	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8982	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8983	{
8984	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8985	AssertRC(rc);
8986	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8987	}
8988	}
8989	}
8990
8991
8992	/**
8993	* Runs the guest code using VT-x the normal way.
8994	*
8995	* @returns VBox status code.
8996	* @param pVM The cross context VM structure.
8997	* @param pVCpu The cross context virtual CPU structure.
8998	* @param pCtx Pointer to the guest-CPU context.
8999	*
9000	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9001	*/
9002	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9003	{
9004	VMXTRANSIENT VmxTransient;
9005	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9006	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9007	uint32_t cLoops = 0;
9008
9009	for (;; cLoops++)
9010	{
9011	Assert(!HMR0SuspendPending());
9012	HMVMX_ASSERT_CPU_SAFE();
9013
9014	/* Preparatory work for running guest code, this may force us to return
9015	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9016	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9017	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9018	if (rcStrict != VINF_SUCCESS)
9019	break;
9020
9021	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9022	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9023	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9024
9025	/* Restore any residual host-state and save any bits shared between host
9026	and guest into the guest-CPU state. Re-enables interrupts! */
9027	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9028
9029	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9030	if (RT_SUCCESS(rcRun))
9031	{ /* very likely */ }
9032	else
9033	{
9034	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9035	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9036	return rcRun;
9037	}
9038
9039	/* Profile the VM-exit. */
9040	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9041	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9042	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9043	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9044	HMVMX_START_EXIT_DISPATCH_PROF();
9045
9046	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9047
9048	/* Handle the VM-exit. */
9049	#ifdef HMVMX_USE_FUNCTION_TABLE
9050	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9051	#else
9052	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9053	#endif
9054	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9055	if (rcStrict == VINF_SUCCESS)
9056	{
9057	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9058	continue; /* likely */
9059	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9060	rcStrict = VINF_EM_RAW_INTERRUPT;
9061	}
9062	break;
9063	}
9064
9065	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9066	return rcStrict;
9067	}
9068
9069
9070
9071	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9072	* probes.
9073	*
9074	* The following few functions and associated structure contains the bloat
9075	* necessary for providing detailed debug events and dtrace probes as well as
9076	* reliable host side single stepping. This works on the principle of
9077	* "subclassing" the normal execution loop and workers. We replace the loop
9078	* method completely and override selected helpers to add necessary adjustments
9079	* to their core operation.
9080	*
9081	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9082	* any performance for debug and analysis features.
9083	*
9084	* @{
9085	*/
9086
9087	/**
9088	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9089	* the debug run loop.
9090	*/
9091	typedef struct VMXRUNDBGSTATE
9092	{
9093	/** The RIP we started executing at. This is for detecting that we stepped. */
9094	uint64_t uRipStart;
9095	/** The CS we started executing with. */
9096	uint16_t uCsStart;
9097
9098	/** Whether we've actually modified the 1st execution control field. */
9099	bool fModifiedProcCtls : 1;
9100	/** Whether we've actually modified the 2nd execution control field. */
9101	bool fModifiedProcCtls2 : 1;
9102	/** Whether we've actually modified the exception bitmap. */
9103	bool fModifiedXcptBitmap : 1;
9104
9105	/** We desire the modified the CR0 mask to be cleared. */
9106	bool fClearCr0Mask : 1;
9107	/** We desire the modified the CR4 mask to be cleared. */
9108	bool fClearCr4Mask : 1;
9109	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9110	uint32_t fCpe1Extra;
9111	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9112	uint32_t fCpe1Unwanted;
9113	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9114	uint32_t fCpe2Extra;
9115	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9116	uint32_t bmXcptExtra;
9117	/** The sequence number of the Dtrace provider settings the state was
9118	* configured against. */
9119	uint32_t uDtraceSettingsSeqNo;
9120	/** VM-exits to check (one bit per VM-exit). */
9121	uint32_t bmExitsToCheck[3];
9122
9123	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9124	uint32_t fProcCtlsInitial;
9125	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9126	uint32_t fProcCtls2Initial;
9127	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9128	uint32_t bmXcptInitial;
9129	} VMXRUNDBGSTATE;
9130	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9131	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9132
9133
9134	/**
9135	* Initializes the VMXRUNDBGSTATE structure.
9136	*
9137	* @param pVCpu The cross context virtual CPU structure of the
9138	* calling EMT.
9139	* @param pCtx The CPU register context to go with @a pVCpu.
9140	* @param pDbgState The structure to initialize.
9141	*/
9142	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9143	{
9144	pDbgState->uRipStart = pCtx->rip;
9145	pDbgState->uCsStart = pCtx->cs.Sel;
9146
9147	pDbgState->fModifiedProcCtls = false;
9148	pDbgState->fModifiedProcCtls2 = false;
9149	pDbgState->fModifiedXcptBitmap = false;
9150	pDbgState->fClearCr0Mask = false;
9151	pDbgState->fClearCr4Mask = false;
9152	pDbgState->fCpe1Extra = 0;
9153	pDbgState->fCpe1Unwanted = 0;
9154	pDbgState->fCpe2Extra = 0;
9155	pDbgState->bmXcptExtra = 0;
9156	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9157	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9158	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9159	}
9160
9161
9162	/**
9163	* Updates the VMSC fields with changes requested by @a pDbgState.
9164	*
9165	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9166	* immediately before executing guest code, i.e. when interrupts are disabled.
9167	* We don't check status codes here as we cannot easily assert or return in the
9168	* latter case.
9169	*
9170	* @param pVCpu The cross context virtual CPU structure.
9171	* @param pDbgState The debug state.
9172	*/
9173	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9174	{
9175	/*
9176	* Ensure desired flags in VMCS control fields are set.
9177	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9178	*
9179	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9180	* there should be no stale data in pCtx at this point.
9181	*/
9182	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9183	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9184	{
9185	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9186	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9187	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9188	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9189	pDbgState->fModifiedProcCtls = true;
9190	}
9191
9192	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9193	{
9194	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9195	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9196	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9197	pDbgState->fModifiedProcCtls2 = true;
9198	}
9199
9200	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9201	{
9202	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9203	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9204	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9205	pDbgState->fModifiedXcptBitmap = true;
9206	}
9207
9208	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9209	{
9210	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9211	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9212	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9213	}
9214
9215	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9216	{
9217	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9218	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9219	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9220	}
9221	}
9222
9223
9224	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9225	{
9226	/*
9227	* Restore VM-exit control settings as we may not reenter this function the
9228	* next time around.
9229	*/
9230	/* We reload the initial value, trigger what we can of recalculations the
9231	next time around. From the looks of things, that's all that's required atm. */
9232	if (pDbgState->fModifiedProcCtls)
9233	{
9234	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9235	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9236	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9237	AssertRCReturn(rc2, rc2);
9238	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9239	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_DEBUG);
9240	}
9241
9242	/* We're currently the only ones messing with this one, so just restore the
9243	cached value and reload the field. */
9244	if ( pDbgState->fModifiedProcCtls2
9245	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9246	{
9247	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9248	AssertRCReturn(rc2, rc2);
9249	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9250	}
9251
9252	/* If we've modified the exception bitmap, we restore it and trigger
9253	reloading and partial recalculation the next time around. */
9254	if (pDbgState->fModifiedXcptBitmap)
9255	{
9256	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9257	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS \| HM_CHANGED_GUEST_CR0);
9258	}
9259
9260	/* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9261	if (pDbgState->fClearCr0Mask)
9262	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9263
9264	/* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9265	if (pDbgState->fClearCr4Mask)
9266	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9267
9268	return rcStrict;
9269	}
9270
9271
9272	/**
9273	* Configures VM-exit controls for current DBGF and DTrace settings.
9274	*
9275	* This updates @a pDbgState and the VMCS execution control fields to reflect
9276	* the necessary VM-exits demanded by DBGF and DTrace.
9277	*
9278	* @param pVM The cross context VM structure.
9279	* @param pVCpu The cross context virtual CPU structure.
9280	* @param pCtx Pointer to the guest-CPU context.
9281	* @param pDbgState The debug state.
9282	* @param pVmxTransient Pointer to the VMX transient structure. May update
9283	* fUpdateTscOffsettingAndPreemptTimer.
9284	*/
9285	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9286	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9287	{
9288	/*
9289	* Take down the dtrace serial number so we can spot changes.
9290	*/
9291	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9292	ASMCompilerBarrier();
9293
9294	/*
9295	* We'll rebuild most of the middle block of data members (holding the
9296	* current settings) as we go along here, so start by clearing it all.
9297	*/
9298	pDbgState->bmXcptExtra = 0;
9299	pDbgState->fCpe1Extra = 0;
9300	pDbgState->fCpe1Unwanted = 0;
9301	pDbgState->fCpe2Extra = 0;
9302	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9303	pDbgState->bmExitsToCheck[i] = 0;
9304
9305	/*
9306	* Software interrupts (INT XXh) - no idea how to trigger these...
9307	*/
9308	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9309	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9310	{
9311	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9312	}
9313
9314	/*
9315	* INT3 breakpoints - triggered by #BP exceptions.
9316	*/
9317	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9318	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9319
9320	/*
9321	* Exception bitmap and XCPT events+probes.
9322	*/
9323	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9324	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9325	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9326
9327	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9328	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9329	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9330	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9331	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9332	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9333	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9334	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9335	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9336	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9337	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9338	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9339	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9340	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9341	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9342	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9343	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9344	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9345
9346	if (pDbgState->bmXcptExtra)
9347	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9348
9349	/*
9350	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9351	*
9352	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9353	* So, when adding/changing/removing please don't forget to update it.
9354	*
9355	* Some of the macros are picking up local variables to save horizontal space,
9356	* (being able to see it in a table is the lesser evil here).
9357	*/
9358	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9359	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9360	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9361	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9362	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9363	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9364	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9365	} else do { } while (0)
9366	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9367	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9368	{ \
9369	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9370	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9371	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9372	} else do { } while (0)
9373	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9374	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9375	{ \
9376	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9377	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9378	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9379	} else do { } while (0)
9380	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9381	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9382	{ \
9383	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9384	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9385	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9386	} else do { } while (0)
9387
9388	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9389	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9390	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9391	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9392	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9393
9394	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9395	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9396	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9397	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9398	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9399	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9400	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9401	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9402	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9403	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9404	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9405	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9406	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9407	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9408	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9409	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9410	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9411	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9412	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9413	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9414	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9415	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9416	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9417	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9418	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9419	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9420	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9421	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9422	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9423	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9424	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9425	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9426	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9427	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9428	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9430
9431	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9432	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9433	{
9434	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9435	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9436	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9437	AssertRC(rc2);
9438
9439	#if 0 /** @todo fix me */
9440	pDbgState->fClearCr0Mask = true;
9441	pDbgState->fClearCr4Mask = true;
9442	#endif
9443	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9444	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9445	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9446	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9447	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9448	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9449	require clearing here and in the loop if we start using it. */
9450	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9451	}
9452	else
9453	{
9454	if (pDbgState->fClearCr0Mask)
9455	{
9456	pDbgState->fClearCr0Mask = false;
9457	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9458	}
9459	if (pDbgState->fClearCr4Mask)
9460	{
9461	pDbgState->fClearCr4Mask = false;
9462	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9463	}
9464	}
9465	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9466	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9467
9468	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9469	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9470	{
9471	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9472	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9473	}
9474	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9475	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9476
9477	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9478	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9479	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9480	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9481	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9482	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9483	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9484	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9485	#if 0 /** @todo too slow, fix handler. */
9486	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9487	#endif
9488	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9489
9490	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9491	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9492	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9493	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9494	{
9495	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9496	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9497	}
9498	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9499	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9500	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9501	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9502
9503	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9504	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9505	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9506	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9507	{
9508	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9509	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9510	}
9511	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9512	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9513	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9514	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9515
9516	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9517	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9518	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9519	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9520	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9521	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9522	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9523	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9524	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9525	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9526	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9527	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9528	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9529	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9530	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9531	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9532	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9533	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9534	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9535	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9536	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9537	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9538
9539	#undef IS_EITHER_ENABLED
9540	#undef SET_ONLY_XBM_IF_EITHER_EN
9541	#undef SET_CPE1_XBM_IF_EITHER_EN
9542	#undef SET_CPEU_XBM_IF_EITHER_EN
9543	#undef SET_CPE2_XBM_IF_EITHER_EN
9544
9545	/*
9546	* Sanitize the control stuff.
9547	*/
9548	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9549	if (pDbgState->fCpe2Extra)
9550	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9551	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9552	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9553	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9554	{
9555	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9556	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9557	}
9558
9559	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9560	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9561	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9562	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9563	}
9564
9565
9566	/**
9567	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9568	* appropriate.
9569	*
9570	* The caller has checked the VM-exit against the
9571	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9572	* already, so we don't have to do that either.
9573	*
9574	* @returns Strict VBox status code (i.e. informational status codes too).
9575	* @param pVM The cross context VM structure.
9576	* @param pVCpu The cross context virtual CPU structure.
9577	* @param pMixedCtx Pointer to the guest-CPU context.
9578	* @param pVmxTransient Pointer to the VMX-transient structure.
9579	* @param uExitReason The VM-exit reason.
9580	*
9581	* @remarks The name of this function is displayed by dtrace, so keep it short
9582	* and to the point. No longer than 33 chars long, please.
9583	*/
9584	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9585	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9586	{
9587	/*
9588	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9589	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9590	*
9591	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9592	* does. Must add/change/remove both places. Same ordering, please.
9593	*
9594	* Added/removed events must also be reflected in the next section
9595	* where we dispatch dtrace events.
9596	*/
9597	bool fDtrace1 = false;
9598	bool fDtrace2 = false;
9599	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9600	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9601	uint32_t uEventArg = 0;
9602	#define SET_EXIT(a_EventSubName) \
9603	do { \
9604	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9605	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9606	} while (0)
9607	#define SET_BOTH(a_EventSubName) \
9608	do { \
9609	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9610	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9611	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9612	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9613	} while (0)
9614	switch (uExitReason)
9615	{
9616	case VMX_EXIT_MTF:
9617	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9618
9619	case VMX_EXIT_XCPT_OR_NMI:
9620	{
9621	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9622	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9623	{
9624	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9625	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9626	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9627	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9628	{
9629	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9630	{
9631	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9632	uEventArg = pVmxTransient->uExitIntErrorCode;
9633	}
9634	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9635	switch (enmEvent1)
9636	{
9637	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9638	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9639	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9640	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9641	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9642	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9643	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9644	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9645	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9646	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9647	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9648	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9649	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9650	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9651	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9652	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9653	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9654	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9655	default: break;
9656	}
9657	}
9658	else
9659	AssertFailed();
9660	break;
9661
9662	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9663	uEventArg = idxVector;
9664	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9665	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9666	break;
9667	}
9668	break;
9669	}
9670
9671	case VMX_EXIT_TRIPLE_FAULT:
9672	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9673	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9674	break;
9675	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9676	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9677	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9678	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9679	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9680
9681	/* Instruction specific VM-exits: */
9682	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9683	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9684	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9685	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9686	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9687	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9688	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9689	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9690	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9691	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9692	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9693	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9694	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9695	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9696	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9697	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9698	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9699	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9700	case VMX_EXIT_MOV_CRX:
9701	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9702	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9703	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9704	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9705	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9706	SET_BOTH(CRX_READ);
9707	else
9708	SET_BOTH(CRX_WRITE);
9709	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9710	break;
9711	case VMX_EXIT_MOV_DRX:
9712	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9713	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9714	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9715	SET_BOTH(DRX_READ);
9716	else
9717	SET_BOTH(DRX_WRITE);
9718	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9719	break;
9720	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9721	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9722	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9723	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9724	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9725	case VMX_EXIT_XDTR_ACCESS:
9726	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9727	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9728	{
9729	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9730	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9731	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9732	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9733	}
9734	break;
9735
9736	case VMX_EXIT_TR_ACCESS:
9737	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9738	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9739	{
9740	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9741	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9742	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9743	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9744	}
9745	break;
9746
9747	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9748	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9749	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9750	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9751	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9752	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9753	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9754	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9755	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9756	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9757	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9758
9759	/* Events that aren't relevant at this point. */
9760	case VMX_EXIT_EXT_INT:
9761	case VMX_EXIT_INT_WINDOW:
9762	case VMX_EXIT_NMI_WINDOW:
9763	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9764	case VMX_EXIT_PREEMPT_TIMER:
9765	case VMX_EXIT_IO_INSTR:
9766	break;
9767
9768	/* Errors and unexpected events. */
9769	case VMX_EXIT_INIT_SIGNAL:
9770	case VMX_EXIT_SIPI:
9771	case VMX_EXIT_IO_SMI:
9772	case VMX_EXIT_SMI:
9773	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9774	case VMX_EXIT_ERR_MSR_LOAD:
9775	case VMX_EXIT_ERR_MACHINE_CHECK:
9776	break;
9777
9778	default:
9779	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9780	break;
9781	}
9782	#undef SET_BOTH
9783	#undef SET_EXIT
9784
9785	/*
9786	* Dtrace tracepoints go first. We do them here at once so we don't
9787	* have to copy the guest state saving and stuff a few dozen times.
9788	* Down side is that we've got to repeat the switch, though this time
9789	* we use enmEvent since the probes are a subset of what DBGF does.
9790	*/
9791	if (fDtrace1 \|\| fDtrace2)
9792	{
9793	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9794	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9795	switch (enmEvent1)
9796	{
9797	/** @todo consider which extra parameters would be helpful for each probe. */
9798	case DBGFEVENT_END: break;
9799	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9800	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9801	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9802	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9803	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9804	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9805	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9806	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9807	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9808	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9809	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9810	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9811	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9812	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9813	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9814	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9815	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9816	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9817	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9818	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9819	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9820	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9821	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9822	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9823	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9824	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9825	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9826	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9827	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9828	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9829	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9830	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9831	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9832	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9833	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9834	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9835	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9836	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9837	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9838	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9839	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9840	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9841	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9842	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9843	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9844	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9845	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9846	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9847	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9848	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9849	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9850	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9851	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9852	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9853	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9854	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9855	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9856	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9857	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9858	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9859	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9860	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9861	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9862	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9863	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9864	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9865	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9866	}
9867	switch (enmEvent2)
9868	{
9869	/** @todo consider which extra parameters would be helpful for each probe. */
9870	case DBGFEVENT_END: break;
9871	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9872	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9873	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9874	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9875	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9876	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9877	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9878	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9879	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9880	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9881	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9882	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9883	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9884	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9885	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9886	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9887	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9888	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9889	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9890	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9891	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9892	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9893	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9894	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9895	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9896	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9897	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9898	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9899	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9900	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9901	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9902	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9903	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9904	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9905	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9906	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9907	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9908	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9909	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9910	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9911	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9912	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9913	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9914	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9915	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9916	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9917	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9918	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9919	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9920	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9921	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9922	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9923	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9924	}
9925	}
9926
9927	/*
9928	* Fire of the DBGF event, if enabled (our check here is just a quick one,
9929	* the DBGF call will do a full check).
9930	*
9931	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9932	* Note! If we have to events, we prioritize the first, i.e. the instruction
9933	* one, in order to avoid event nesting.
9934	*/
9935	if ( enmEvent1 != DBGFEVENT_END
9936	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9937	{
9938	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9939	if (rcStrict != VINF_SUCCESS)
9940	return rcStrict;
9941	}
9942	else if ( enmEvent2 != DBGFEVENT_END
9943	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9944	{
9945	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9946	if (rcStrict != VINF_SUCCESS)
9947	return rcStrict;
9948	}
9949
9950	return VINF_SUCCESS;
9951	}
9952
9953
9954	/**
9955	* Single-stepping VM-exit filtering.
9956	*
9957	* This is preprocessing the VM-exits and deciding whether we've gotten far
9958	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9959	* handling is performed.
9960	*
9961	* @returns Strict VBox status code (i.e. informational status codes too).
9962	* @param pVM The cross context VM structure.
9963	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9964	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9965	* out-of-sync. Make sure to update the required
9966	* fields before using them.
9967	* @param pVmxTransient Pointer to the VMX-transient structure.
9968	* @param uExitReason The VM-exit reason.
9969	* @param pDbgState The debug state.
9970	*/
9971	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9972	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9973	{
9974	/*
9975	* Expensive (saves context) generic dtrace VM-exit probe.
9976	*/
9977	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9978	{ /* more likely */ }
9979	else
9980	{
9981	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9982	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9983	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9984	}
9985
9986	/*
9987	* Check for host NMI, just to get that out of the way.
9988	*/
9989	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9990	{ /* normally likely */ }
9991	else
9992	{
9993	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9994	AssertRCReturn(rc2, rc2);
9995	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9996	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9997	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9998	}
9999
10000	/*
10001	* Check for single stepping event if we're stepping.
10002	*/
10003	if (pVCpu->hm.s.fSingleInstruction)
10004	{
10005	switch (uExitReason)
10006	{
10007	case VMX_EXIT_MTF:
10008	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10009
10010	/* Various events: */
10011	case VMX_EXIT_XCPT_OR_NMI:
10012	case VMX_EXIT_EXT_INT:
10013	case VMX_EXIT_TRIPLE_FAULT:
10014	case VMX_EXIT_INT_WINDOW:
10015	case VMX_EXIT_NMI_WINDOW:
10016	case VMX_EXIT_TASK_SWITCH:
10017	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10018	case VMX_EXIT_APIC_ACCESS:
10019	case VMX_EXIT_EPT_VIOLATION:
10020	case VMX_EXIT_EPT_MISCONFIG:
10021	case VMX_EXIT_PREEMPT_TIMER:
10022
10023	/* Instruction specific VM-exits: */
10024	case VMX_EXIT_CPUID:
10025	case VMX_EXIT_GETSEC:
10026	case VMX_EXIT_HLT:
10027	case VMX_EXIT_INVD:
10028	case VMX_EXIT_INVLPG:
10029	case VMX_EXIT_RDPMC:
10030	case VMX_EXIT_RDTSC:
10031	case VMX_EXIT_RSM:
10032	case VMX_EXIT_VMCALL:
10033	case VMX_EXIT_VMCLEAR:
10034	case VMX_EXIT_VMLAUNCH:
10035	case VMX_EXIT_VMPTRLD:
10036	case VMX_EXIT_VMPTRST:
10037	case VMX_EXIT_VMREAD:
10038	case VMX_EXIT_VMRESUME:
10039	case VMX_EXIT_VMWRITE:
10040	case VMX_EXIT_VMXOFF:
10041	case VMX_EXIT_VMXON:
10042	case VMX_EXIT_MOV_CRX:
10043	case VMX_EXIT_MOV_DRX:
10044	case VMX_EXIT_IO_INSTR:
10045	case VMX_EXIT_RDMSR:
10046	case VMX_EXIT_WRMSR:
10047	case VMX_EXIT_MWAIT:
10048	case VMX_EXIT_MONITOR:
10049	case VMX_EXIT_PAUSE:
10050	case VMX_EXIT_XDTR_ACCESS:
10051	case VMX_EXIT_TR_ACCESS:
10052	case VMX_EXIT_INVEPT:
10053	case VMX_EXIT_RDTSCP:
10054	case VMX_EXIT_INVVPID:
10055	case VMX_EXIT_WBINVD:
10056	case VMX_EXIT_XSETBV:
10057	case VMX_EXIT_RDRAND:
10058	case VMX_EXIT_INVPCID:
10059	case VMX_EXIT_VMFUNC:
10060	case VMX_EXIT_RDSEED:
10061	case VMX_EXIT_XSAVES:
10062	case VMX_EXIT_XRSTORS:
10063	{
10064	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10065	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10066	AssertRCReturn(rc2, rc2);
10067	if ( pMixedCtx->rip != pDbgState->uRipStart
10068	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10069	return VINF_EM_DBG_STEPPED;
10070	break;
10071	}
10072
10073	/* Errors and unexpected events: */
10074	case VMX_EXIT_INIT_SIGNAL:
10075	case VMX_EXIT_SIPI:
10076	case VMX_EXIT_IO_SMI:
10077	case VMX_EXIT_SMI:
10078	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10079	case VMX_EXIT_ERR_MSR_LOAD:
10080	case VMX_EXIT_ERR_MACHINE_CHECK:
10081	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10082	break;
10083
10084	default:
10085	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10086	break;
10087	}
10088	}
10089
10090	/*
10091	* Check for debugger event breakpoints and dtrace probes.
10092	*/
10093	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10094	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10095	{
10096	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10097	if (rcStrict != VINF_SUCCESS)
10098	return rcStrict;
10099	}
10100
10101	/*
10102	* Normal processing.
10103	*/
10104	#ifdef HMVMX_USE_FUNCTION_TABLE
10105	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10106	#else
10107	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10108	#endif
10109	}
10110
10111
10112	/**
10113	* Single steps guest code using VT-x.
10114	*
10115	* @returns Strict VBox status code (i.e. informational status codes too).
10116	* @param pVM The cross context VM structure.
10117	* @param pVCpu The cross context virtual CPU structure.
10118	* @param pCtx Pointer to the guest-CPU context.
10119	*
10120	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10121	*/
10122	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10123	{
10124	VMXTRANSIENT VmxTransient;
10125	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10126
10127	/* Set HMCPU indicators. */
10128	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10129	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10130	pVCpu->hm.s.fDebugWantRdTscExit = false;
10131	pVCpu->hm.s.fUsingDebugLoop = true;
10132
10133	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10134	VMXRUNDBGSTATE DbgState;
10135	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10136	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10137
10138	/*
10139	* The loop.
10140	*/
10141	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10142	for (uint32_t cLoops = 0; ; cLoops++)
10143	{
10144	Assert(!HMR0SuspendPending());
10145	HMVMX_ASSERT_CPU_SAFE();
10146	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10147
10148	/*
10149	* Preparatory work for running guest code, this may force us to return
10150	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10151	*/
10152	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10153	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10154	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10155	if (rcStrict != VINF_SUCCESS)
10156	break;
10157
10158	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10159	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10160
10161	/*
10162	* Now we can run the guest code.
10163	*/
10164	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10165
10166	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10167
10168	/*
10169	* Restore any residual host-state and save any bits shared between host
10170	* and guest into the guest-CPU state. Re-enables interrupts!
10171	*/
10172	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10173
10174	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10175	if (RT_SUCCESS(rcRun))
10176	{ /* very likely */ }
10177	else
10178	{
10179	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10180	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10181	return rcRun;
10182	}
10183
10184	/* Profile the VM-exit. */
10185	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10187	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10188	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10189	HMVMX_START_EXIT_DISPATCH_PROF();
10190
10191	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10192
10193	/*
10194	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10195	*/
10196	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10197	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10198	if (rcStrict != VINF_SUCCESS)
10199	break;
10200	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10201	{
10202	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10203	rcStrict = VINF_EM_RAW_INTERRUPT;
10204	break;
10205	}
10206
10207	/*
10208	* Stepping: Did the RIP change, if so, consider it a single step.
10209	* Otherwise, make sure one of the TFs gets set.
10210	*/
10211	if (fStepping)
10212	{
10213	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10214	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10215	AssertRCReturn(rc2, rc2);
10216	if ( pCtx->rip != DbgState.uRipStart
10217	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10218	{
10219	rcStrict = VINF_EM_DBG_STEPPED;
10220	break;
10221	}
10222	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10223	}
10224
10225	/*
10226	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10227	*/
10228	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10229	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10230	}
10231
10232	/*
10233	* Clear the X86_EFL_TF if necessary.
10234	*/
10235	if (pVCpu->hm.s.fClearTrapFlag)
10236	{
10237	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10238	AssertRCReturn(rc2, rc2);
10239	pVCpu->hm.s.fClearTrapFlag = false;
10240	pCtx->eflags.Bits.u1TF = 0;
10241	}
10242	/** @todo there seems to be issues with the resume flag when the monitor trap
10243	* flag is pending without being used. Seen early in bios init when
10244	* accessing APIC page in protected mode. */
10245
10246	/*
10247	* Restore VM-exit control settings as we may not reenter this function the
10248	* next time around.
10249	*/
10250	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10251
10252	/* Restore HMCPU indicators. */
10253	pVCpu->hm.s.fUsingDebugLoop = false;
10254	pVCpu->hm.s.fDebugWantRdTscExit = false;
10255	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10256
10257	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10258	return rcStrict;
10259	}
10260
10261
10262	/** @} */
10263
10264
10265	/**
10266	* Checks if any expensive dtrace probes are enabled and we should go to the
10267	* debug loop.
10268	*
10269	* @returns true if we should use debug loop, false if not.
10270	*/
10271	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10272	{
10273	/* It's probably faster to OR the raw 32-bit counter variables together.
10274	Since the variables are in an array and the probes are next to one
10275	another (more or less), we have good locality. So, better read
10276	eight-nine cache lines ever time and only have one conditional, than
10277	128+ conditionals, right? */
10278	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10279	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10280	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10281	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10282	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10283	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10284	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10285	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10286	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10287	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10288	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10289	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10290	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10291	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10292	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10293	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10294	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10295	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10296	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10297	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10298	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10299	) != 0
10300	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10301	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10302	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10303	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10304	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10305	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10306	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10307	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10308	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10309	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10310	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10311	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10312	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10313	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10314	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10315	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10316	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10317	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10318	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10319	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10320	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10321	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10322	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10323	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10324	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10325	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10326	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10327	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10328	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10329	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10330	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10331	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10332	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10333	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10334	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10335	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10336	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10337	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10338	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10339	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10340	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10341	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10342	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10343	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10344	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10345	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10346	) != 0
10347	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10348	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10349	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10350	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10351	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10352	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10353	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10354	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10355	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10356	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10357	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10358	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10359	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10360	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10361	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10362	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10363	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10364	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10365	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10366	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10367	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10368	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10369	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10370	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10371	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10372	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10373	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10374	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10375	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10376	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10377	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10378	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10379	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10380	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10381	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10382	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10383	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10384	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10385	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10386	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10387	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10388	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10389	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10390	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10391	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10392	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10393	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10394	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10395	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10396	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10397	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10398	) != 0;
10399	}
10400
10401
10402	/**
10403	* Runs the guest code using VT-x.
10404	*
10405	* @returns Strict VBox status code (i.e. informational status codes too).
10406	* @param pVM The cross context VM structure.
10407	* @param pVCpu The cross context virtual CPU structure.
10408	* @param pCtx Pointer to the guest-CPU context.
10409	*/
10410	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10411	{
10412	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10413	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10414	HMVMX_ASSERT_PREEMPT_SAFE();
10415
10416	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10417
10418	VBOXSTRICTRC rcStrict;
10419	if ( !pVCpu->hm.s.fUseDebugLoop
10420	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10421	&& !DBGFIsStepping(pVCpu)
10422	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10423	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10424	else
10425	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10426
10427	if (rcStrict == VERR_EM_INTERPRETER)
10428	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10429	else if (rcStrict == VINF_EM_RESET)
10430	rcStrict = VINF_EM_TRIPLE_FAULT;
10431
10432	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10433	if (RT_FAILURE(rc2))
10434	{
10435	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10436	rcStrict = rc2;
10437	}
10438	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10439	return rcStrict;
10440	}
10441
10442
10443	#ifndef HMVMX_USE_FUNCTION_TABLE
10444	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10445	{
10446	# ifdef DEBUG_ramshankar
10447	# define RETURN_EXIT_CALL(a_CallExpr) \
10448	do { \
10449	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10450	VBOXSTRICTRC rcStrict = a_CallExpr; \
10451	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10452	return rcStrict; \
10453	} while (0)
10454	# else
10455	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10456	# endif
10457	switch (rcReason)
10458	{
10459	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10460	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10461	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10462	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10463	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10464	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10465	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10466	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10467	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10468	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10469	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10470	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10471	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10472	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10473	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10474	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10475	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10476	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10477	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10478	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10479	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10480	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10481	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10482	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10483	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10484	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10485	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10486	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10487	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10488	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10489	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10490	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10491	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10492	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10493
10494	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10495	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10496	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10497	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10498	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10499	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10500	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10501	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10502	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10503
10504	case VMX_EXIT_VMCLEAR:
10505	case VMX_EXIT_VMLAUNCH:
10506	case VMX_EXIT_VMPTRLD:
10507	case VMX_EXIT_VMPTRST:
10508	case VMX_EXIT_VMREAD:
10509	case VMX_EXIT_VMRESUME:
10510	case VMX_EXIT_VMWRITE:
10511	case VMX_EXIT_VMXOFF:
10512	case VMX_EXIT_VMXON:
10513	case VMX_EXIT_INVEPT:
10514	case VMX_EXIT_INVVPID:
10515	case VMX_EXIT_VMFUNC:
10516	case VMX_EXIT_XSAVES:
10517	case VMX_EXIT_XRSTORS:
10518	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10519	case VMX_EXIT_ENCLS:
10520	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10521	case VMX_EXIT_PML_FULL:
10522	default:
10523	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10524	}
10525	#undef RETURN_EXIT_CALL
10526	}
10527	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10528
10529
10530	#ifdef VBOX_STRICT
10531	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10532	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10533	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10534
10535	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10536	do { \
10537	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10538	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10539	} while (0)
10540
10541	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10542	do { \
10543	AssertPtr(pVCpu); \
10544	AssertPtr(pMixedCtx); \
10545	AssertPtr(pVmxTransient); \
10546	Assert(pVmxTransient->fVMEntryFailed == false); \
10547	Assert(ASMIntAreEnabled()); \
10548	HMVMX_ASSERT_PREEMPT_SAFE(); \
10549	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10550	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10551	HMVMX_ASSERT_PREEMPT_SAFE(); \
10552	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10553	HMVMX_ASSERT_PREEMPT_CPUID(); \
10554	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10555	} while (0)
10556
10557	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10558	do { \
10559	Log4Func(("\n")); \
10560	} while (0)
10561	#else /* nonstrict builds: */
10562	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10563	do { \
10564	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10565	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10566	} while (0)
10567	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10568	#endif
10569
10570
10571	/**
10572	* Advances the guest RIP by the specified number of bytes.
10573	*
10574	* @param pVCpu The cross context virtual CPU structure.
10575	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10576	* out-of-sync. Make sure to update the required fields
10577	* before using them.
10578	* @param cbInstr Number of bytes to advance the RIP by.
10579	*
10580	* @remarks No-long-jump zone!!!
10581	*/
10582	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10583	{
10584	/* Advance the RIP. */
10585	pMixedCtx->rip += cbInstr;
10586	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10587
10588	/* Update interrupt inhibition. */
10589	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10590	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10591	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10592	}
10593
10594
10595	/**
10596	* Advances the guest RIP after reading it from the VMCS.
10597	*
10598	* @returns VBox status code, no informational status codes.
10599	* @param pVCpu The cross context virtual CPU structure.
10600	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10601	* out-of-sync. Make sure to update the required fields
10602	* before using them.
10603	* @param pVmxTransient Pointer to the VMX transient structure.
10604	*
10605	* @remarks No-long-jump zone!!!
10606	*/
10607	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10608	{
10609	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10610	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10611	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10612	AssertRCReturn(rc, rc);
10613
10614	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10615
10616	/*
10617	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10618	* pending debug exception field as it takes care of priority of events.
10619	*
10620	* See Intel spec. 32.2.1 "Debug Exceptions".
10621	*/
10622	if ( !pVCpu->hm.s.fSingleInstruction
10623	&& pMixedCtx->eflags.Bits.u1TF)
10624	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10625
10626	return VINF_SUCCESS;
10627	}
10628
10629
10630	/**
10631	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10632	* and update error record fields accordingly.
10633	*
10634	* @return VMX_IGS_* return codes.
10635	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10636	* wrong with the guest state.
10637	*
10638	* @param pVM The cross context VM structure.
10639	* @param pVCpu The cross context virtual CPU structure.
10640	* @param pCtx Pointer to the guest-CPU state.
10641	*
10642	* @remarks This function assumes our cache of the VMCS controls
10643	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10644	*/
10645	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10646	{
10647	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10648	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10649	uError = (err); \
10650	break; \
10651	} else do { } while (0)
10652
10653	int rc;
10654	uint32_t uError = VMX_IGS_ERROR;
10655	uint32_t u32Val;
10656	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10657
10658	do
10659	{
10660	/*
10661	* CR0.
10662	*/
10663	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10664	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10665	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10666	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10667	if (fUnrestrictedGuest)
10668	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10669
10670	uint32_t u32GuestCR0;
10671	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10672	AssertRCBreak(rc);
10673	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10674	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10675	if ( !fUnrestrictedGuest
10676	&& (u32GuestCR0 & X86_CR0_PG)
10677	&& !(u32GuestCR0 & X86_CR0_PE))
10678	{
10679	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10680	}
10681
10682	/*
10683	* CR4.
10684	*/
10685	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10686	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10687
10688	uint32_t u32GuestCR4;
10689	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10690	AssertRCBreak(rc);
10691	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10692	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10693
10694	/*
10695	* IA32_DEBUGCTL MSR.
10696	*/
10697	uint64_t u64Val;
10698	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10699	AssertRCBreak(rc);
10700	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10701	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10702	{
10703	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10704	}
10705	uint64_t u64DebugCtlMsr = u64Val;
10706
10707	#ifdef VBOX_STRICT
10708	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10709	AssertRCBreak(rc);
10710	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10711	#endif
10712	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10713
10714	/*
10715	* RIP and RFLAGS.
10716	*/
10717	uint32_t u32Eflags;
10718	#if HC_ARCH_BITS == 64
10719	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10720	AssertRCBreak(rc);
10721	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10722	if ( !fLongModeGuest
10723	\|\| !pCtx->cs.Attr.n.u1Long)
10724	{
10725	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10726	}
10727	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10728	* must be identical if the "IA-32e mode guest" VM-entry
10729	* control is 1 and CS.L is 1. No check applies if the
10730	* CPU supports 64 linear-address bits. */
10731
10732	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10733	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10734	AssertRCBreak(rc);
10735	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10736	VMX_IGS_RFLAGS_RESERVED);
10737	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10738	u32Eflags = u64Val;
10739	#else
10740	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10741	AssertRCBreak(rc);
10742	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10743	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10744	#endif
10745
10746	if ( fLongModeGuest
10747	\|\| ( fUnrestrictedGuest
10748	&& !(u32GuestCR0 & X86_CR0_PE)))
10749	{
10750	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10751	}
10752
10753	uint32_t u32EntryInfo;
10754	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10755	AssertRCBreak(rc);
10756	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10757	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10758	{
10759	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10760	}
10761
10762	/*
10763	* 64-bit checks.
10764	*/
10765	#if HC_ARCH_BITS == 64
10766	if (fLongModeGuest)
10767	{
10768	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10769	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10770	}
10771
10772	if ( !fLongModeGuest
10773	&& (u32GuestCR4 & X86_CR4_PCIDE))
10774	{
10775	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10776	}
10777
10778	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10779	* 51:32 beyond the processor's physical-address width are 0. */
10780
10781	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10782	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10783	{
10784	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10785	}
10786
10787	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10788	AssertRCBreak(rc);
10789	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10790
10791	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10792	AssertRCBreak(rc);
10793	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10794	#endif
10795
10796	/*
10797	* PERF_GLOBAL MSR.
10798	*/
10799	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10800	{
10801	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10802	AssertRCBreak(rc);
10803	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10804	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10805	}
10806
10807	/*
10808	* PAT MSR.
10809	*/
10810	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10811	{
10812	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10813	AssertRCBreak(rc);
10814	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10815	for (unsigned i = 0; i < 8; i++)
10816	{
10817	uint8_t u8Val = (u64Val & 0xff);
10818	if ( u8Val != 0 /* UC */
10819	&& u8Val != 1 /* WC */
10820	&& u8Val != 4 /* WT */
10821	&& u8Val != 5 /* WP */
10822	&& u8Val != 6 /* WB */
10823	&& u8Val != 7 /* UC- */)
10824	{
10825	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10826	}
10827	u64Val >>= 8;
10828	}
10829	}
10830
10831	/*
10832	* EFER MSR.
10833	*/
10834	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10835	{
10836	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10837	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10838	AssertRCBreak(rc);
10839	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10840	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10841	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10842	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10843	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10844	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10845	\|\| !(u32GuestCR0 & X86_CR0_PG)
10846	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10847	VMX_IGS_EFER_LMA_LME_MISMATCH);
10848	}
10849
10850	/*
10851	* Segment registers.
10852	*/
10853	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10854	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10855	if (!(u32Eflags & X86_EFL_VM))
10856	{
10857	/* CS */
10858	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10859	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10860	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10861	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10862	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10863	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10864	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10865	/* CS cannot be loaded with NULL in protected mode. */
10866	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10867	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10868	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10869	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10870	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10871	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10872	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10873	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10874	else
10875	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10876
10877	/* SS */
10878	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10879	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10880	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10881	if ( !(pCtx->cr0 & X86_CR0_PE)
10882	\|\| pCtx->cs.Attr.n.u4Type == 3)
10883	{
10884	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10885	}
10886	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10887	{
10888	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10889	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10890	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10891	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10892	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10893	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10894	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10895	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10896	}
10897
10898	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10899	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10900	{
10901	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10902	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10903	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10904	\|\| pCtx->ds.Attr.n.u4Type > 11
10905	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10906	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10907	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10908	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10909	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10910	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10911	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10912	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10913	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10914	}
10915	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10916	{
10917	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10918	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10919	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10920	\|\| pCtx->es.Attr.n.u4Type > 11
10921	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10922	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10923	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10924	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10925	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10926	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10927	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10928	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10929	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10930	}
10931	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10932	{
10933	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10934	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10935	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10936	\|\| pCtx->fs.Attr.n.u4Type > 11
10937	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10938	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10939	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10940	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10941	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10942	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10943	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10944	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10945	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10946	}
10947	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10948	{
10949	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10950	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10951	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10952	\|\| pCtx->gs.Attr.n.u4Type > 11
10953	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10954	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10955	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10956	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10957	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10958	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10959	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10960	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10961	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10962	}
10963	/* 64-bit capable CPUs. */
10964	#if HC_ARCH_BITS == 64
10965	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10966	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10967	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10968	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10969	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10970	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
10971	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10972	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
10973	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10974	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
10975	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10976	#endif
10977	}
10978	else
10979	{
10980	/* V86 mode checks. */
10981	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10982	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10983	{
10984	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10985	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10986	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10987	}
10988	else
10989	{
10990	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10991	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10992	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10993	}
10994
10995	/* CS */
10996	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10997	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10998	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10999	/* SS */
11000	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11001	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11002	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11003	/* DS */
11004	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11005	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11006	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11007	/* ES */
11008	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11009	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11010	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11011	/* FS */
11012	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11013	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11014	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11015	/* GS */
11016	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11017	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11018	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11019	/* 64-bit capable CPUs. */
11020	#if HC_ARCH_BITS == 64
11021	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11022	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11023	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11024	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11025	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11026	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11027	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11028	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11029	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11030	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11031	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11032	#endif
11033	}
11034
11035	/*
11036	* TR.
11037	*/
11038	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11039	/* 64-bit capable CPUs. */
11040	#if HC_ARCH_BITS == 64
11041	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11042	#endif
11043	if (fLongModeGuest)
11044	{
11045	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11046	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11047	}
11048	else
11049	{
11050	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11051	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11052	VMX_IGS_TR_ATTR_TYPE_INVALID);
11053	}
11054	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11055	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11056	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11057	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11058	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11059	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11060	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11061	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11062
11063	/*
11064	* GDTR and IDTR.
11065	*/
11066	#if HC_ARCH_BITS == 64
11067	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11068	AssertRCBreak(rc);
11069	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11070
11071	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11072	AssertRCBreak(rc);
11073	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11074	#endif
11075
11076	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11077	AssertRCBreak(rc);
11078	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11079
11080	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11081	AssertRCBreak(rc);
11082	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11083
11084	/*
11085	* Guest Non-Register State.
11086	*/
11087	/* Activity State. */
11088	uint32_t u32ActivityState;
11089	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11090	AssertRCBreak(rc);
11091	HMVMX_CHECK_BREAK( !u32ActivityState
11092	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11093	VMX_IGS_ACTIVITY_STATE_INVALID);
11094	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11095	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11096	uint32_t u32IntrState;
11097	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11098	AssertRCBreak(rc);
11099	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11100	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11101	{
11102	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11103	}
11104
11105	/** @todo Activity state and injecting interrupts. Left as a todo since we
11106	* currently don't use activity states but ACTIVE. */
11107
11108	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11109	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11110
11111	/* Guest interruptibility-state. */
11112	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11113	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11114	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11115	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11116	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11117	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11118	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11119	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11120	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11121	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11122	{
11123	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11124	{
11125	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11126	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11127	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11128	}
11129	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11130	{
11131	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11132	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11133	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11134	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11135	}
11136	}
11137	/** @todo Assumes the processor is not in SMM. */
11138	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11139	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11140	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11141	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11142	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11143	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11144	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11145	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11146	{
11147	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11148	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11149	}
11150
11151	/* Pending debug exceptions. */
11152	#if HC_ARCH_BITS == 64
11153	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11154	AssertRCBreak(rc);
11155	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11156	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11157	u32Val = u64Val; /* For pending debug exceptions checks below. */
11158	#else
11159	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11160	AssertRCBreak(rc);
11161	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11162	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11163	#endif
11164
11165	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11166	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11167	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11168	{
11169	if ( (u32Eflags & X86_EFL_TF)
11170	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11171	{
11172	/* Bit 14 is PendingDebug.BS. */
11173	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11174	}
11175	if ( !(u32Eflags & X86_EFL_TF)
11176	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11177	{
11178	/* Bit 14 is PendingDebug.BS. */
11179	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11180	}
11181	}
11182
11183	/* VMCS link pointer. */
11184	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11185	AssertRCBreak(rc);
11186	if (u64Val != UINT64_C(0xffffffffffffffff))
11187	{
11188	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11189	/** @todo Bits beyond the processor's physical-address width MBZ. */
11190	/** @todo 32-bit located in memory referenced by value of this field (as a
11191	* physical address) must contain the processor's VMCS revision ID. */
11192	/** @todo SMM checks. */
11193	}
11194
11195	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11196	* not using Nested Paging? */
11197	if ( pVM->hm.s.fNestedPaging
11198	&& !fLongModeGuest
11199	&& CPUMIsGuestInPAEModeEx(pCtx))
11200	{
11201	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11202	AssertRCBreak(rc);
11203	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11204
11205	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11206	AssertRCBreak(rc);
11207	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11208
11209	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11210	AssertRCBreak(rc);
11211	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11212
11213	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11214	AssertRCBreak(rc);
11215	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11216	}
11217
11218	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11219	if (uError == VMX_IGS_ERROR)
11220	uError = VMX_IGS_REASON_NOT_FOUND;
11221	} while (0);
11222
11223	pVCpu->hm.s.u32HMError = uError;
11224	return uError;
11225
11226	#undef HMVMX_ERROR_BREAK
11227	#undef HMVMX_CHECK_BREAK
11228	}
11229
11230	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11231	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11232	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11233
11234	/** @name VM-exit handlers.
11235	* @{
11236	*/
11237
11238	/**
11239	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11240	*/
11241	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11242	{
11243	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11244	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11245	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11246	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11247	return VINF_SUCCESS;
11248	return VINF_EM_RAW_INTERRUPT;
11249	}
11250
11251
11252	/**
11253	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11254	*/
11255	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11256	{
11257	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11258	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11259
11260	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11261	AssertRCReturn(rc, rc);
11262
11263	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11264	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11265	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11266	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11267
11268	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11269	{
11270	/*
11271	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11272	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11273	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11274	*
11275	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11276	*/
11277	VMXDispatchHostNmi();
11278	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11279	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11280	return VINF_SUCCESS;
11281	}
11282
11283	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11284	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11285	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11286	{ /* likely */ }
11287	else
11288	{
11289	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11290	rcStrictRc1 = VINF_SUCCESS;
11291	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11292	return rcStrictRc1;
11293	}
11294
11295	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11296	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11297	switch (uIntType)
11298	{
11299	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11300	Assert(uVector == X86_XCPT_DB);
11301	/* no break */
11302	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11303	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11304	/* no break */
11305	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11306	{
11307	/*
11308	* If there's any exception caused as a result of event injection, go back to
11309	* the interpreter. The page-fault case is complicated and we manually handle
11310	* any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11311	* handled in hmR0VmxCheckExitDueToEventDelivery.
11312	*/
11313	if (!pVCpu->hm.s.Event.fPending)
11314	{ /* likely */ }
11315	else if ( uVector != X86_XCPT_PF
11316	&& uVector != X86_XCPT_AC)
11317	{
11318	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11319	rc = VERR_EM_INTERPRETER;
11320	break;
11321	}
11322
11323	switch (uVector)
11324	{
11325	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11326	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11327	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11328	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11329	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11330	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11331	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11332
11333	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11334	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11335	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11336	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11337	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11338	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11339	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11340	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11341	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11342	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11343	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11344	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11345	default:
11346	{
11347	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11348	AssertRCReturn(rc, rc);
11349
11350	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11351	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11352	{
11353	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11354	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11355	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11356
11357	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11358	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11359	AssertRCReturn(rc, rc);
11360	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11361	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11362	0 /* GCPtrFaultAddress */);
11363	AssertRCReturn(rc, rc);
11364	}
11365	else
11366	{
11367	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11368	pVCpu->hm.s.u32HMError = uVector;
11369	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11370	}
11371	break;
11372	}
11373	}
11374	break;
11375	}
11376
11377	default:
11378	{
11379	pVCpu->hm.s.u32HMError = uExitIntInfo;
11380	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11381	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11382	break;
11383	}
11384	}
11385	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11386	return rc;
11387	}
11388
11389
11390	/**
11391	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11392	*/
11393	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11394	{
11395	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11396
11397	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11398	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11399
11400	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11401	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11402	return VINF_SUCCESS;
11403	}
11404
11405
11406	/**
11407	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11408	*/
11409	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11410	{
11411	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11412	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11413	{
11414	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11415	HMVMX_RETURN_UNEXPECTED_EXIT();
11416	}
11417
11418	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11419
11420	/*
11421	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11422	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11423	*/
11424	uint32_t uIntrState = 0;
11425	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11426	AssertRCReturn(rc, rc);
11427
11428	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11429	if ( fBlockSti
11430	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11431	{
11432	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11433	}
11434
11435	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11436	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11437
11438	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11439	return VINF_SUCCESS;
11440	}
11441
11442
11443	/**
11444	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11445	*/
11446	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11447	{
11448	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11450	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11451	}
11452
11453
11454	/**
11455	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11456	*/
11457	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11458	{
11459	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11460	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11461	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11462	}
11463
11464
11465	/**
11466	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11467	*/
11468	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11469	{
11470	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11471	PVM pVM = pVCpu->CTX_SUFF(pVM);
11472	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11473	if (RT_LIKELY(rc == VINF_SUCCESS))
11474	{
11475	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11476	Assert(pVmxTransient->cbInstr == 2);
11477	}
11478	else
11479	{
11480	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11481	rc = VERR_EM_INTERPRETER;
11482	}
11483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11484	return rc;
11485	}
11486
11487
11488	/**
11489	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11490	*/
11491	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11492	{
11493	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11494	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11495	AssertRCReturn(rc, rc);
11496
11497	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11498	return VINF_EM_RAW_EMULATE_INSTR;
11499
11500	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11501	HMVMX_RETURN_UNEXPECTED_EXIT();
11502	}
11503
11504
11505	/**
11506	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11507	*/
11508	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11509	{
11510	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11511	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11512	AssertRCReturn(rc, rc);
11513
11514	PVM pVM = pVCpu->CTX_SUFF(pVM);
11515	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11516	if (RT_LIKELY(rc == VINF_SUCCESS))
11517	{
11518	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11519	Assert(pVmxTransient->cbInstr == 2);
11520	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11521	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11522	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11523	}
11524	else
11525	rc = VERR_EM_INTERPRETER;
11526	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11527	return rc;
11528	}
11529
11530
11531	/**
11532	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11533	*/
11534	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535	{
11536	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11537	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11538	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11539	AssertRCReturn(rc, rc);
11540
11541	PVM pVM = pVCpu->CTX_SUFF(pVM);
11542	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11543	if (RT_SUCCESS(rc))
11544	{
11545	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11546	Assert(pVmxTransient->cbInstr == 3);
11547	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11548	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11549	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11550	}
11551	else
11552	{
11553	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11554	rc = VERR_EM_INTERPRETER;
11555	}
11556	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11557	return rc;
11558	}
11559
11560
11561	/**
11562	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11563	*/
11564	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11565	{
11566	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11567	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11568	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11569	AssertRCReturn(rc, rc);
11570
11571	PVM pVM = pVCpu->CTX_SUFF(pVM);
11572	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11573	if (RT_LIKELY(rc == VINF_SUCCESS))
11574	{
11575	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11576	Assert(pVmxTransient->cbInstr == 2);
11577	}
11578	else
11579	{
11580	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11581	rc = VERR_EM_INTERPRETER;
11582	}
11583	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11584	return rc;
11585	}
11586
11587
11588	/**
11589	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11590	*/
11591	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11592	{
11593	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11595
11596	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11597	if (pVCpu->hm.s.fHypercallsEnabled)
11598	{
11599	#if 0
11600	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11601	#else
11602	/* Aggressive state sync. for now. */
11603	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11604	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11605	AssertRCReturn(rc, rc);
11606	#endif
11607
11608	/* Perform the hypercall. */
11609	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11610	if (rcStrict == VINF_SUCCESS)
11611	{
11612	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11613	AssertRCReturn(rc, rc);
11614	}
11615	else
11616	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11617	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11618	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11619
11620	/* If the hypercall changes anything other than guest's general-purpose registers,
11621	we would need to reload the guest changed bits here before VM-entry. */
11622	}
11623	else
11624	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11625
11626	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11627	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11628	{
11629	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11630	rcStrict = VINF_SUCCESS;
11631	}
11632
11633	return rcStrict;
11634	}
11635
11636
11637	/**
11638	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11639	*/
11640	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11641	{
11642	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11643	PVM pVM = pVCpu->CTX_SUFF(pVM);
11644	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11645
11646	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11647	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11648	AssertRCReturn(rc, rc);
11649
11650	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11651	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11652	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11653	else
11654	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11655	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11656	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11657	return rcStrict;
11658	}
11659
11660
11661	/**
11662	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11663	*/
11664	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11665	{
11666	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11667	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11668	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11669	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11670	AssertRCReturn(rc, rc);
11671
11672	PVM pVM = pVCpu->CTX_SUFF(pVM);
11673	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11674	if (RT_LIKELY(rc == VINF_SUCCESS))
11675	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11676	else
11677	{
11678	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11679	rc = VERR_EM_INTERPRETER;
11680	}
11681	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11682	return rc;
11683	}
11684
11685
11686	/**
11687	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11688	*/
11689	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11690	{
11691	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11692	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11693	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11694	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11695	AssertRCReturn(rc, rc);
11696
11697	PVM pVM = pVCpu->CTX_SUFF(pVM);
11698	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11699	rc = VBOXSTRICTRC_VAL(rc2);
11700	if (RT_LIKELY( rc == VINF_SUCCESS
11701	\|\| rc == VINF_EM_HALT))
11702	{
11703	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11704	AssertRCReturn(rc3, rc3);
11705
11706	if ( rc == VINF_EM_HALT
11707	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11708	{
11709	rc = VINF_SUCCESS;
11710	}
11711	}
11712	else
11713	{
11714	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11715	rc = VERR_EM_INTERPRETER;
11716	}
11717	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11718	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11719	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11720	return rc;
11721	}
11722
11723
11724	/**
11725	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11726	*/
11727	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11728	{
11729	/*
11730	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11731	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11732	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
11733	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11734	*/
11735	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11736	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11737	HMVMX_RETURN_UNEXPECTED_EXIT();
11738	}
11739
11740
11741	/**
11742	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11743	*/
11744	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11745	{
11746	/*
11747	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11748	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11749	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
11750	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11751	*/
11752	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11753	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11754	HMVMX_RETURN_UNEXPECTED_EXIT();
11755	}
11756
11757
11758	/**
11759	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11760	*/
11761	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11762	{
11763	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11764	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11765	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11766	HMVMX_RETURN_UNEXPECTED_EXIT();
11767	}
11768
11769
11770	/**
11771	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11772	*/
11773	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11774	{
11775	/*
11776	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11777	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11778	* See Intel spec. 25.3 "Other Causes of VM-exits".
11779	*/
11780	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11781	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11782	HMVMX_RETURN_UNEXPECTED_EXIT();
11783	}
11784
11785
11786	/**
11787	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11788	* VM-exit.
11789	*/
11790	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11791	{
11792	/*
11793	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11794	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11795	*
11796	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11797	* See Intel spec. "23.8 Restrictions on VMX operation".
11798	*/
11799	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11800	return VINF_SUCCESS;
11801	}
11802
11803
11804	/**
11805	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11806	* VM-exit.
11807	*/
11808	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11809	{
11810	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11811	return VINF_EM_RESET;
11812	}
11813
11814
11815	/**
11816	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11817	*/
11818	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11819	{
11820	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11821	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11822
11823	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11824	AssertRCReturn(rc, rc);
11825
11826	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11827	rc = VINF_SUCCESS;
11828	else
11829	rc = VINF_EM_HALT;
11830
11831	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11832	if (rc != VINF_SUCCESS)
11833	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11834	return rc;
11835	}
11836
11837
11838	/**
11839	* VM-exit handler for instructions that result in a \#UD exception delivered to
11840	* the guest.
11841	*/
11842	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11843	{
11844	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11845	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11846	return VINF_SUCCESS;
11847	}
11848
11849
11850	/**
11851	* VM-exit handler for expiry of the VMX preemption timer.
11852	*/
11853	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11854	{
11855	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11856
11857	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11858	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11859
11860	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11861	PVM pVM = pVCpu->CTX_SUFF(pVM);
11862	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11863	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11864	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11865	}
11866
11867
11868	/**
11869	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11870	*/
11871	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11872	{
11873	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11874
11875	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11876	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
11877	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11878	AssertRCReturn(rc, rc);
11879
11880	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11881	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11882
11883	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11884
11885	return rcStrict;
11886	}
11887
11888
11889	/**
11890	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11891	*/
11892	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11893	{
11894	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11895
11896	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11897	/** @todo implement EMInterpretInvpcid() */
11898	return VERR_EM_INTERPRETER;
11899	}
11900
11901
11902	/**
11903	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11904	* Error VM-exit.
11905	*/
11906	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11907	{
11908	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11909	AssertRCReturn(rc, rc);
11910
11911	rc = hmR0VmxCheckVmcsCtls(pVCpu);
11912	AssertRCReturn(rc, rc);
11913
11914	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11915	NOREF(uInvalidReason);
11916
11917	#ifdef VBOX_STRICT
11918	uint32_t uIntrState;
11919	RTHCUINTREG uHCReg;
11920	uint64_t u64Val;
11921	uint32_t u32Val;
11922
11923	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11924	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11925	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11926	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11927	AssertRCReturn(rc, rc);
11928
11929	Log4(("uInvalidReason %u\n", uInvalidReason));
11930	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11931	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11932	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11933	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11934
11935	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11936	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11937	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11938	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11939	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11940	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11941	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11942	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11943	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11944	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11945	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11946	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11947	#else
11948	NOREF(pVmxTransient);
11949	#endif
11950
11951	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11952	return VERR_VMX_INVALID_GUEST_STATE;
11953	}
11954
11955
11956	/**
11957	* VM-exit handler for VM-entry failure due to an MSR-load
11958	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11959	*/
11960	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11961	{
11962	NOREF(pVmxTransient);
11963	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11964	HMVMX_RETURN_UNEXPECTED_EXIT();
11965	}
11966
11967
11968	/**
11969	* VM-exit handler for VM-entry failure due to a machine-check event
11970	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11971	*/
11972	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11973	{
11974	NOREF(pVmxTransient);
11975	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11976	HMVMX_RETURN_UNEXPECTED_EXIT();
11977	}
11978
11979
11980	/**
11981	* VM-exit handler for all undefined reasons. Should never ever happen.. in
11982	* theory.
11983	*/
11984	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11985	{
11986	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11987	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11988	return VERR_VMX_UNDEFINED_EXIT_CODE;
11989	}
11990
11991
11992	/**
11993	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11994	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11995	* Conditional VM-exit.
11996	*/
11997	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11998	{
11999	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12000
12001	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12002	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12003	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12004	return VERR_EM_INTERPRETER;
12005	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12006	HMVMX_RETURN_UNEXPECTED_EXIT();
12007	}
12008
12009
12010	/**
12011	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12012	*/
12013	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12014	{
12015	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12016
12017	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12019	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12020	return VERR_EM_INTERPRETER;
12021	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12022	HMVMX_RETURN_UNEXPECTED_EXIT();
12023	}
12024
12025
12026	/**
12027	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12028	*/
12029	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12030	{
12031	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12032
12033	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12034	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12035	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12036	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12037	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12038	{
12039	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12040	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12041	}
12042	AssertRCReturn(rc, rc);
12043	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12044
12045	#ifdef VBOX_STRICT
12046	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12047	{
12048	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12049	&& pMixedCtx->ecx != MSR_K6_EFER)
12050	{
12051	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12052	pMixedCtx->ecx));
12053	HMVMX_RETURN_UNEXPECTED_EXIT();
12054	}
12055	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12056	{
12057	VMXMSREXITREAD enmRead;
12058	VMXMSREXITWRITE enmWrite;
12059	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12060	AssertRCReturn(rc2, rc2);
12061	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12062	{
12063	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12064	HMVMX_RETURN_UNEXPECTED_EXIT();
12065	}
12066	}
12067	}
12068	#endif
12069
12070	PVM pVM = pVCpu->CTX_SUFF(pVM);
12071	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12072	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12073	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12074	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12075	if (RT_SUCCESS(rc))
12076	{
12077	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12078	Assert(pVmxTransient->cbInstr == 2);
12079	}
12080	return rc;
12081	}
12082
12083
12084	/**
12085	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12086	*/
12087	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12088	{
12089	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12090	PVM pVM = pVCpu->CTX_SUFF(pVM);
12091	int rc = VINF_SUCCESS;
12092
12093	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12094	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12095	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12096	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12097	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12098	{
12099	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12100	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12101	}
12102	AssertRCReturn(rc, rc);
12103	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12104
12105	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12106	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12107	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12108
12109	if (RT_SUCCESS(rc))
12110	{
12111	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12112
12113	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12114	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12115	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12116	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12117	{
12118	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12119	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12120	EMInterpretWrmsr() changes it. */
12121	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12122	}
12123	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12124	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12125	else if (pMixedCtx->ecx == MSR_K6_EFER)
12126	{
12127	/*
12128	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12129	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12130	* the other bits as well, SCE and NXE. See @bugref{7368}.
12131	*/
12132	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12133	}
12134
12135	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12136	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12137	{
12138	switch (pMixedCtx->ecx)
12139	{
12140	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12141	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12142	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12143	case MSR_K8_FS_BASE: /* no break */
12144	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12145	case MSR_K6_EFER: /* already handled above */ break;
12146	default:
12147	{
12148	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12149	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12150	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12151	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12152	break;
12153	}
12154	}
12155	}
12156	#ifdef VBOX_STRICT
12157	else
12158	{
12159	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12160	switch (pMixedCtx->ecx)
12161	{
12162	case MSR_IA32_SYSENTER_CS:
12163	case MSR_IA32_SYSENTER_EIP:
12164	case MSR_IA32_SYSENTER_ESP:
12165	case MSR_K8_FS_BASE:
12166	case MSR_K8_GS_BASE:
12167	{
12168	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12169	HMVMX_RETURN_UNEXPECTED_EXIT();
12170	}
12171
12172	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12173	default:
12174	{
12175	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12176	{
12177	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12178	if (pMixedCtx->ecx != MSR_K6_EFER)
12179	{
12180	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12181	pMixedCtx->ecx));
12182	HMVMX_RETURN_UNEXPECTED_EXIT();
12183	}
12184	}
12185
12186	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12187	{
12188	VMXMSREXITREAD enmRead;
12189	VMXMSREXITWRITE enmWrite;
12190	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12191	AssertRCReturn(rc2, rc2);
12192	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12193	{
12194	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12195	HMVMX_RETURN_UNEXPECTED_EXIT();
12196	}
12197	}
12198	break;
12199	}
12200	}
12201	}
12202	#endif /* VBOX_STRICT */
12203	}
12204	return rc;
12205	}
12206
12207
12208	/**
12209	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12210	*/
12211	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12212	{
12213	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12214
12215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12216	return VINF_EM_RAW_INTERRUPT;
12217	}
12218
12219
12220	/**
12221	* VM-exit handler for when the TPR value is lowered below the specified
12222	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12223	*/
12224	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12225	{
12226	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12227	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12228
12229	/*
12230	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12231	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12232	*/
12233	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12234	return VINF_SUCCESS;
12235	}
12236
12237
12238	/**
12239	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12240	* VM-exit.
12241	*
12242	* @retval VINF_SUCCESS when guest execution can continue.
12243	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12244	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12245	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12246	* interpreter.
12247	*/
12248	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12249	{
12250	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12251	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12252	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12253	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12254	AssertRCReturn(rc, rc);
12255
12256	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12257	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12258	PVM pVM = pVCpu->CTX_SUFF(pVM);
12259	VBOXSTRICTRC rcStrict;
12260	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12261	switch (uAccessType)
12262	{
12263	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12264	{
12265	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12266	AssertRCReturn(rc, rc);
12267
12268	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12269	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12270	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12271	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12272	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12273	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12274	{
12275	case 0: /* CR0 */
12276	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12277	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12278	break;
12279	case 2: /* CR2 */
12280	/* Nothing to do here, CR2 it's not part of the VMCS. */
12281	break;
12282	case 3: /* CR3 */
12283	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12284	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12285	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12286	break;
12287	case 4: /* CR4 */
12288	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12289	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12290	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12291	break;
12292	case 8: /* CR8 */
12293	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12294	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12295	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12296	break;
12297	default:
12298	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12299	break;
12300	}
12301
12302	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12303	break;
12304	}
12305
12306	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12307	{
12308	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12309	AssertRCReturn(rc, rc);
12310
12311	Assert( !pVM->hm.s.fNestedPaging
12312	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12313	\|\| pVCpu->hm.s.fUsingDebugLoop
12314	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12315
12316	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12317	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12318	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12319
12320	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12321	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12322	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12323	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12325	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12326	VBOXSTRICTRC_VAL(rcStrict)));
12327	break;
12328	}
12329
12330	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12331	{
12332	AssertRCReturn(rc, rc);
12333	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12334	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12335	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12337	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12338	break;
12339	}
12340
12341	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12342	{
12343	AssertRCReturn(rc, rc);
12344	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12345	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12346	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12347	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12348	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12349	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12350	break;
12351	}
12352
12353	default:
12354	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12355	VERR_VMX_UNEXPECTED_EXCEPTION);
12356	}
12357
12358	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12359	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12360	NOREF(pVM);
12361	return rcStrict;
12362	}
12363
12364
12365	/**
12366	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12367	* VM-exit.
12368	*/
12369	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12370	{
12371	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12372	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12373
12374	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12375	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12376	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12377	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12378	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12379	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12380	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12381	AssertRCReturn(rc2, rc2);
12382
12383	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12384	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12385	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12386	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12387	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12388	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12389	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12390	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12391	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12392
12393	/* I/O operation lookup arrays. */
12394	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12395	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12396
12397	VBOXSTRICTRC rcStrict;
12398	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12399	uint32_t const cbInstr = pVmxTransient->cbInstr;
12400	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12401	PVM pVM = pVCpu->CTX_SUFF(pVM);
12402	if (fIOString)
12403	{
12404	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12405	See @bugref{5752#c158}. Should work now. */
12406	/*
12407	* INS/OUTS - I/O String instruction.
12408	*
12409	* Use instruction-information if available, otherwise fall back on
12410	* interpreting the instruction.
12411	*/
12412	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12413	fIOWrite ? 'w' : 'r'));
12414	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12415	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12416	{
12417	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12418	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12419	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12420	AssertRCReturn(rc2, rc2);
12421	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12422	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12423	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12424	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12425	if (fIOWrite)
12426	{
12427	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12428	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12429	}
12430	else
12431	{
12432	/*
12433	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12434	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12435	* See Intel Instruction spec. for "INS".
12436	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12437	*/
12438	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12439	}
12440	}
12441	else
12442	{
12443	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12444	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12445	AssertRCReturn(rc2, rc2);
12446	rcStrict = IEMExecOne(pVCpu);
12447	}
12448	/** @todo IEM needs to be setting these flags somehow. */
12449	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12450	fUpdateRipAlready = true;
12451	#else
12452	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12453	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12454	if (RT_SUCCESS(rcStrict))
12455	{
12456	if (fIOWrite)
12457	{
12458	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12459	(DISCPUMODE)pDis->uAddrMode, cbValue);
12460	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12461	}
12462	else
12463	{
12464	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12465	(DISCPUMODE)pDis->uAddrMode, cbValue);
12466	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12467	}
12468	}
12469	else
12470	{
12471	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12472	pMixedCtx->rip));
12473	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12474	}
12475	#endif
12476	}
12477	else
12478	{
12479	/*
12480	* IN/OUT - I/O instruction.
12481	*/
12482	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12483	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12484	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12485	if (fIOWrite)
12486	{
12487	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12488	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12489	}
12490	else
12491	{
12492	uint32_t u32Result = 0;
12493	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12494	if (IOM_SUCCESS(rcStrict))
12495	{
12496	/* Save result of I/O IN instr. in AL/AX/EAX. */
12497	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12498	}
12499	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12500	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12501	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12502	}
12503	}
12504
12505	if (IOM_SUCCESS(rcStrict))
12506	{
12507	if (!fUpdateRipAlready)
12508	{
12509	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12510	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12511	}
12512
12513	/*
12514	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12515	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12516	*/
12517	if (fIOString)
12518	{
12519	/** @todo Single-step for INS/OUTS with REP prefix? */
12520	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12521	}
12522	else if ( !fDbgStepping
12523	&& fGstStepping)
12524	{
12525	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12526	}
12527
12528	/*
12529	* If any I/O breakpoints are armed, we need to check if one triggered
12530	* and take appropriate action.
12531	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12532	*/
12533	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12534	AssertRCReturn(rc2, rc2);
12535
12536	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12537	* execution engines about whether hyper BPs and such are pending. */
12538	uint32_t const uDr7 = pMixedCtx->dr[7];
12539	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12540	&& X86_DR7_ANY_RW_IO(uDr7)
12541	&& (pMixedCtx->cr4 & X86_CR4_DE))
12542	\|\| DBGFBpIsHwIoArmed(pVM)))
12543	{
12544	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12545
12546	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12547	VMMRZCallRing3Disable(pVCpu);
12548	HM_DISABLE_PREEMPT();
12549
12550	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12551
12552	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12553	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12554	{
12555	/* Raise #DB. */
12556	if (fIsGuestDbgActive)
12557	ASMSetDR6(pMixedCtx->dr[6]);
12558	if (pMixedCtx->dr[7] != uDr7)
12559	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12560
12561	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12562	}
12563	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12564	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12565	else if ( rcStrict2 != VINF_SUCCESS
12566	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12567	rcStrict = rcStrict2;
12568	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12569
12570	HM_RESTORE_PREEMPT();
12571	VMMRZCallRing3Enable(pVCpu);
12572	}
12573	}
12574
12575	#ifdef VBOX_STRICT
12576	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12577	Assert(!fIOWrite);
12578	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12579	Assert(fIOWrite);
12580	else
12581	{
12582	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12583	* statuses, that the VMM device and some others may return. See
12584	* IOM_SUCCESS() for guidance. */
12585	AssertMsg( RT_FAILURE(rcStrict)
12586	\|\| rcStrict == VINF_SUCCESS
12587	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12588	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12589	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12590	\|\| rcStrict == VINF_EM_RAW_TO_R3
12591	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12592	#endif
12593	}
12594	#endif
12595
12596	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12597	return rcStrict;
12598	}
12599
12600
12601	/**
12602	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12603	* VM-exit.
12604	*/
12605	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12606	{
12607	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12608
12609	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12610	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12611	AssertRCReturn(rc, rc);
12612	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12613	{
12614	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12615	AssertRCReturn(rc, rc);
12616	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12617	{
12618	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12619
12620	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12621	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12622
12623	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12624	Assert(!pVCpu->hm.s.Event.fPending);
12625	pVCpu->hm.s.Event.fPending = true;
12626	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12627	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12628	AssertRCReturn(rc, rc);
12629	if (fErrorCodeValid)
12630	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12631	else
12632	pVCpu->hm.s.Event.u32ErrCode = 0;
12633	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12634	&& uVector == X86_XCPT_PF)
12635	{
12636	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12637	}
12638
12639	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12641	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12642	}
12643	}
12644
12645	/* Fall back to the interpreter to emulate the task-switch. */
12646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12647	return VERR_EM_INTERPRETER;
12648	}
12649
12650
12651	/**
12652	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12653	*/
12654	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12655	{
12656	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12657	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12658	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12659	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12660	AssertRCReturn(rc, rc);
12661	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12662	return VINF_EM_DBG_STEPPED;
12663	}
12664
12665
12666	/**
12667	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12668	*/
12669	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12670	{
12671	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12672
12673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12674
12675	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12676	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12677	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12678	{
12679	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12680	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12681	{
12682	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12683	return VERR_EM_INTERPRETER;
12684	}
12685	}
12686	else
12687	{
12688	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12689	rcStrict1 = VINF_SUCCESS;
12690	return rcStrict1;
12691	}
12692
12693	#if 0
12694	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12695	* just sync the whole thing. */
12696	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12697	#else
12698	/* Aggressive state sync. for now. */
12699	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12700	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12701	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12702	#endif
12703	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12704	AssertRCReturn(rc, rc);
12705
12706	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12707	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12708	VBOXSTRICTRC rcStrict2;
12709	switch (uAccessType)
12710	{
12711	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12712	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12713	{
12714	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12715	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12716	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12717
12718	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12719	GCPhys &= PAGE_BASE_GC_MASK;
12720	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12721	PVM pVM = pVCpu->CTX_SUFF(pVM);
12722	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12723	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12724
12725	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12726	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12727	CPUMCTX2CORE(pMixedCtx), GCPhys);
12728	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12729	if ( rcStrict2 == VINF_SUCCESS
12730	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12731	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12732	{
12733	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12734	\| HM_CHANGED_GUEST_RSP
12735	\| HM_CHANGED_GUEST_RFLAGS
12736	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
12737	rcStrict2 = VINF_SUCCESS;
12738	}
12739	break;
12740	}
12741
12742	default:
12743	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12744	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12745	break;
12746	}
12747
12748	if (rcStrict2 != VINF_SUCCESS)
12749	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12750	return rcStrict2;
12751	}
12752
12753
12754	/**
12755	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12756	* VM-exit.
12757	*/
12758	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12759	{
12760	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12761
12762	/* We should -not- get this VM-exit if the guest's debug registers were active. */
12763	if (pVmxTransient->fWasGuestDebugStateActive)
12764	{
12765	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12766	HMVMX_RETURN_UNEXPECTED_EXIT();
12767	}
12768
12769	if ( !pVCpu->hm.s.fSingleInstruction
12770	&& !pVmxTransient->fWasHyperDebugStateActive)
12771	{
12772	Assert(!DBGFIsStepping(pVCpu));
12773	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12774
12775	/* Don't intercept MOV DRx any more. */
12776	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12777	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12778	AssertRCReturn(rc, rc);
12779
12780	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12781	VMMRZCallRing3Disable(pVCpu);
12782	HM_DISABLE_PREEMPT();
12783
12784	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12785	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12786	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
12787
12788	HM_RESTORE_PREEMPT();
12789	VMMRZCallRing3Enable(pVCpu);
12790
12791	#ifdef VBOX_WITH_STATISTICS
12792	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12793	AssertRCReturn(rc, rc);
12794	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12796	else
12797	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12798	#endif
12799	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12800	return VINF_SUCCESS;
12801	}
12802
12803	/*
12804	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12805	* Update the segment registers and DR7 from the CPU.
12806	*/
12807	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12808	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12809	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12810	AssertRCReturn(rc, rc);
12811	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12812
12813	PVM pVM = pVCpu->CTX_SUFF(pVM);
12814	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12815	{
12816	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12817	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12818	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12819	if (RT_SUCCESS(rc))
12820	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12821	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12822	}
12823	else
12824	{
12825	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12826	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12827	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12828	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12829	}
12830
12831	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
12832	if (RT_SUCCESS(rc))
12833	{
12834	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12835	AssertRCReturn(rc2, rc2);
12836	return VINF_SUCCESS;
12837	}
12838	return rc;
12839	}
12840
12841
12842	/**
12843	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12844	* Conditional VM-exit.
12845	*/
12846	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12847	{
12848	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12849	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12850
12851	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12852	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12853	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12854	{
12855	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12856	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12857	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12858	{
12859	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12860	return VERR_EM_INTERPRETER;
12861	}
12862	}
12863	else
12864	{
12865	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12866	rcStrict1 = VINF_SUCCESS;
12867	return rcStrict1;
12868	}
12869
12870	RTGCPHYS GCPhys = 0;
12871	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12872
12873	#if 0
12874	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12875	#else
12876	/* Aggressive state sync. for now. */
12877	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12878	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12879	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12880	#endif
12881	AssertRCReturn(rc, rc);
12882
12883	/*
12884	* If we succeed, resume guest execution.
12885	* If we fail in interpreting the instruction because we couldn't get the guest physical address
12886	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12887	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12888	* weird case. See @bugref{6043}.
12889	*/
12890	PVM pVM = pVCpu->CTX_SUFF(pVM);
12891	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12892	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12893	if ( rcStrict2 == VINF_SUCCESS
12894	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12895	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12896	{
12897	/* Successfully handled MMIO operation. */
12898	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12899	\| HM_CHANGED_GUEST_RSP
12900	\| HM_CHANGED_GUEST_RFLAGS
12901	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
12902	return VINF_SUCCESS;
12903	}
12904	return rcStrict2;
12905	}
12906
12907
12908	/**
12909	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12910	* VM-exit.
12911	*/
12912	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12913	{
12914	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12915	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12916
12917	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12918	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12919	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12920	{
12921	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12922	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12923	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12924	}
12925	else
12926	{
12927	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12928	rcStrict1 = VINF_SUCCESS;
12929	return rcStrict1;
12930	}
12931
12932	RTGCPHYS GCPhys = 0;
12933	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12934	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12935	#if 0
12936	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12937	#else
12938	/* Aggressive state sync. for now. */
12939	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12940	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12941	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12942	#endif
12943	AssertRCReturn(rc, rc);
12944
12945	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12946	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12947
12948	RTGCUINT uErrorCode = 0;
12949	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12950	uErrorCode \|= X86_TRAP_PF_ID;
12951	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12952	uErrorCode \|= X86_TRAP_PF_RW;
12953	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12954	uErrorCode \|= X86_TRAP_PF_P;
12955
12956	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12957
12958	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12959	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12960
12961	/* Handle the pagefault trap for the nested shadow table. */
12962	PVM pVM = pVCpu->CTX_SUFF(pVM);
12963	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12964	TRPMResetTrap(pVCpu);
12965
12966	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12967	if ( rcStrict2 == VINF_SUCCESS
12968	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12969	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12970	{
12971	/* Successfully synced our nested page tables. */
12972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12973	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12974	\| HM_CHANGED_GUEST_RSP
12975	\| HM_CHANGED_GUEST_RFLAGS);
12976	return VINF_SUCCESS;
12977	}
12978
12979	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12980	return rcStrict2;
12981	}
12982
12983	/** @} */
12984
12985	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12986	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12987	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12988
12989	/** @name VM-exit exception handlers.
12990	* @{
12991	*/
12992
12993	/**
12994	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
12995	*/
12996	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12997	{
12998	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12999	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13000
13001	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13002	AssertRCReturn(rc, rc);
13003
13004	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13005	{
13006	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13007	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13008
13009	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13010	* provides VM-exit instruction length. If this causes problem later,
13011	* disassemble the instruction like it's done on AMD-V. */
13012	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13013	AssertRCReturn(rc2, rc2);
13014	return rc;
13015	}
13016
13017	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13018	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13019	return rc;
13020	}
13021
13022
13023	/**
13024	* VM-exit exception handler for \#BP (Breakpoint exception).
13025	*/
13026	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13027	{
13028	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13029	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13030
13031	/** @todo Try optimize this by not saving the entire guest state unless
13032	* really needed. */
13033	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13034	AssertRCReturn(rc, rc);
13035
13036	PVM pVM = pVCpu->CTX_SUFF(pVM);
13037	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13038	if (rc == VINF_EM_RAW_GUEST_TRAP)
13039	{
13040	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13041	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13042	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13043	AssertRCReturn(rc, rc);
13044
13045	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13046	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13047	}
13048
13049	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13050	return rc;
13051	}
13052
13053
13054	/**
13055	* VM-exit exception handler for \#AC (alignment check exception).
13056	*/
13057	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13058	{
13059	RT_NOREF_PV(pMixedCtx);
13060	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13061
13062	/*
13063	* Re-inject it. We'll detect any nesting before getting here.
13064	*/
13065	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13066	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13067	AssertRCReturn(rc, rc);
13068	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13069
13070	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13071	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13072	return VINF_SUCCESS;
13073	}
13074
13075
13076	/**
13077	* VM-exit exception handler for \#DB (Debug exception).
13078	*/
13079	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13080	{
13081	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13082	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13083	Log6(("XcptDB\n"));
13084
13085	/*
13086	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13087	* for processing.
13088	*/
13089	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13090	AssertRCReturn(rc, rc);
13091
13092	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13093	uint64_t uDR6 = X86_DR6_INIT_VAL;
13094	uDR6 \|= ( pVmxTransient->uExitQualification
13095	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13096
13097	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13098	if (rc == VINF_EM_RAW_GUEST_TRAP)
13099	{
13100	/*
13101	* The exception was for the guest. Update DR6, DR7.GD and
13102	* IA32_DEBUGCTL.LBR before forwarding it.
13103	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13104	*/
13105	VMMRZCallRing3Disable(pVCpu);
13106	HM_DISABLE_PREEMPT();
13107
13108	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13109	pMixedCtx->dr[6] \|= uDR6;
13110	if (CPUMIsGuestDebugStateActive(pVCpu))
13111	ASMSetDR6(pMixedCtx->dr[6]);
13112
13113	HM_RESTORE_PREEMPT();
13114	VMMRZCallRing3Enable(pVCpu);
13115
13116	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13117	AssertRCReturn(rc, rc);
13118
13119	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13120	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13121
13122	/* Paranoia. */
13123	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13124	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13125
13126	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13127	AssertRCReturn(rc, rc);
13128
13129	/*
13130	* Raise #DB in the guest.
13131	*
13132	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13133	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13134	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13135	*
13136	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13137	*/
13138	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13139	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13140	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13141	AssertRCReturn(rc, rc);
13142	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13143	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13144	return VINF_SUCCESS;
13145	}
13146
13147	/*
13148	* Not a guest trap, must be a hypervisor related debug event then.
13149	* Update DR6 in case someone is interested in it.
13150	*/
13151	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13152	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13153	CPUMSetHyperDR6(pVCpu, uDR6);
13154
13155	return rc;
13156	}
13157
13158
13159	/**
13160	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13161	* point exception).
13162	*/
13163	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13164	{
13165	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13166
13167	/* We require CR0 and EFER. EFER is always up-to-date. */
13168	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13169	AssertRCReturn(rc, rc);
13170
13171	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13172	VMMRZCallRing3Disable(pVCpu);
13173	HM_DISABLE_PREEMPT();
13174
13175	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13176	if (pVmxTransient->fWasGuestFPUStateActive)
13177	{
13178	rc = VINF_EM_RAW_GUEST_TRAP;
13179	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13180	}
13181	else
13182	{
13183	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13184	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13185	#endif
13186	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13187	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13188	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13189	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13190	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13191	}
13192
13193	HM_RESTORE_PREEMPT();
13194	VMMRZCallRing3Enable(pVCpu);
13195
13196	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13197	{
13198	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13199	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13200	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13201	pVCpu->hm.s.fPreloadGuestFpu = true;
13202	}
13203	else
13204	{
13205	/* Forward #NM to the guest. */
13206	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13207	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13208	AssertRCReturn(rc, rc);
13209	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13210	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13211	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13212	}
13213
13214	return VINF_SUCCESS;
13215	}
13216
13217
13218	/**
13219	* VM-exit exception handler for \#GP (General-protection exception).
13220	*
13221	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13222	*/
13223	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13224	{
13225	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13226	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13227
13228	int rc;
13229	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13230	{ /* likely */ }
13231	else
13232	{
13233	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13234	Assert(pVCpu->hm.s.fUsingDebugLoop);
13235	#endif
13236	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13237	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13238	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13239	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13240	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13241	AssertRCReturn(rc, rc);
13242	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13243	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13244	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13245	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13246	return rc;
13247	}
13248
13249	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13250	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13251
13252	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13253	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13254	AssertRCReturn(rc, rc);
13255
13256	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13257	uint32_t cbOp = 0;
13258	PVM pVM = pVCpu->CTX_SUFF(pVM);
13259	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13260	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13261	if (RT_SUCCESS(rc))
13262	{
13263	rc = VINF_SUCCESS;
13264	Assert(cbOp == pDis->cbInstr);
13265	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13266	switch (pDis->pCurInstr->uOpcode)
13267	{
13268	case OP_CLI:
13269	{
13270	pMixedCtx->eflags.Bits.u1IF = 0;
13271	pMixedCtx->eflags.Bits.u1RF = 0;
13272	pMixedCtx->rip += pDis->cbInstr;
13273	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13274	if ( !fDbgStepping
13275	&& pMixedCtx->eflags.Bits.u1TF)
13276	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13278	break;
13279	}
13280
13281	case OP_STI:
13282	{
13283	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13284	pMixedCtx->eflags.Bits.u1IF = 1;
13285	pMixedCtx->eflags.Bits.u1RF = 0;
13286	pMixedCtx->rip += pDis->cbInstr;
13287	if (!fOldIF)
13288	{
13289	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13290	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13291	}
13292	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13293	if ( !fDbgStepping
13294	&& pMixedCtx->eflags.Bits.u1TF)
13295	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13296	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13297	break;
13298	}
13299
13300	case OP_HLT:
13301	{
13302	rc = VINF_EM_HALT;
13303	pMixedCtx->rip += pDis->cbInstr;
13304	pMixedCtx->eflags.Bits.u1RF = 0;
13305	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13307	break;
13308	}
13309
13310	case OP_POPF:
13311	{
13312	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13313	uint32_t cbParm;
13314	uint32_t uMask;
13315	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13316	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13317	{
13318	cbParm = 4;
13319	uMask = 0xffffffff;
13320	}
13321	else
13322	{
13323	cbParm = 2;
13324	uMask = 0xffff;
13325	}
13326
13327	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13328	RTGCPTR GCPtrStack = 0;
13329	X86EFLAGS Eflags;
13330	Eflags.u32 = 0;
13331	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13332	&GCPtrStack);
13333	if (RT_SUCCESS(rc))
13334	{
13335	Assert(sizeof(Eflags.u32) >= cbParm);
13336	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13337	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13338	}
13339	if (RT_FAILURE(rc))
13340	{
13341	rc = VERR_EM_INTERPRETER;
13342	break;
13343	}
13344	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13345	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13346	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13347	pMixedCtx->esp += cbParm;
13348	pMixedCtx->esp &= uMask;
13349	pMixedCtx->rip += pDis->cbInstr;
13350	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13351	\| HM_CHANGED_GUEST_RSP
13352	\| HM_CHANGED_GUEST_RFLAGS);
13353	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13354	POPF restores EFLAGS.TF. */
13355	if ( !fDbgStepping
13356	&& fGstStepping)
13357	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13358	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13359	break;
13360	}
13361
13362	case OP_PUSHF:
13363	{
13364	uint32_t cbParm;
13365	uint32_t uMask;
13366	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13367	{
13368	cbParm = 4;
13369	uMask = 0xffffffff;
13370	}
13371	else
13372	{
13373	cbParm = 2;
13374	uMask = 0xffff;
13375	}
13376
13377	/* Get the stack pointer & push the contents of eflags onto the stack. */
13378	RTGCPTR GCPtrStack = 0;
13379	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13380	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13381	if (RT_FAILURE(rc))
13382	{
13383	rc = VERR_EM_INTERPRETER;
13384	break;
13385	}
13386	X86EFLAGS Eflags = pMixedCtx->eflags;
13387	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13388	Eflags.Bits.u1RF = 0;
13389	Eflags.Bits.u1VM = 0;
13390
13391	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13392	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13393	{
13394	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13395	rc = VERR_EM_INTERPRETER;
13396	break;
13397	}
13398	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13399	pMixedCtx->esp -= cbParm;
13400	pMixedCtx->esp &= uMask;
13401	pMixedCtx->rip += pDis->cbInstr;
13402	pMixedCtx->eflags.Bits.u1RF = 0;
13403	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13404	\| HM_CHANGED_GUEST_RSP
13405	\| HM_CHANGED_GUEST_RFLAGS);
13406	if ( !fDbgStepping
13407	&& pMixedCtx->eflags.Bits.u1TF)
13408	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13410	break;
13411	}
13412
13413	case OP_IRET:
13414	{
13415	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13416	* instruction reference. */
13417	RTGCPTR GCPtrStack = 0;
13418	uint32_t uMask = 0xffff;
13419	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13420	uint16_t aIretFrame[3];
13421	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13422	{
13423	rc = VERR_EM_INTERPRETER;
13424	break;
13425	}
13426	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13427	&GCPtrStack);
13428	if (RT_SUCCESS(rc))
13429	{
13430	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13431	PGMACCESSORIGIN_HM));
13432	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13433	}
13434	if (RT_FAILURE(rc))
13435	{
13436	rc = VERR_EM_INTERPRETER;
13437	break;
13438	}
13439	pMixedCtx->eip = 0;
13440	pMixedCtx->ip = aIretFrame[0];
13441	pMixedCtx->cs.Sel = aIretFrame[1];
13442	pMixedCtx->cs.ValidSel = aIretFrame[1];
13443	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13444	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13445	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13446	pMixedCtx->sp += sizeof(aIretFrame);
13447	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13448	\| HM_CHANGED_GUEST_SEGMENT_REGS
13449	\| HM_CHANGED_GUEST_RSP
13450	\| HM_CHANGED_GUEST_RFLAGS);
13451	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13452	if ( !fDbgStepping
13453	&& fGstStepping)
13454	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13455	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13456	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13457	break;
13458	}
13459
13460	case OP_INT:
13461	{
13462	uint16_t uVector = pDis->Param1.uValue & 0xff;
13463	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13464	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13465	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13466	break;
13467	}
13468
13469	case OP_INTO:
13470	{
13471	if (pMixedCtx->eflags.Bits.u1OF)
13472	{
13473	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13474	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13476	}
13477	else
13478	{
13479	pMixedCtx->eflags.Bits.u1RF = 0;
13480	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13481	}
13482	break;
13483	}
13484
13485	default:
13486	{
13487	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13488	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13489	EMCODETYPE_SUPERVISOR);
13490	rc = VBOXSTRICTRC_VAL(rc2);
13491	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13492	/** @todo We have to set pending-debug exceptions here when the guest is
13493	* single-stepping depending on the instruction that was interpreted. */
13494	Log4(("#GP rc=%Rrc\n", rc));
13495	break;
13496	}
13497	}
13498	}
13499	else
13500	rc = VERR_EM_INTERPRETER;
13501
13502	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13503	("#GP Unexpected rc=%Rrc\n", rc));
13504	return rc;
13505	}
13506
13507
13508	/**
13509	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13510	* the exception reported in the VMX transient structure back into the VM.
13511	*
13512	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13513	* up-to-date.
13514	*/
13515	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13516	{
13517	RT_NOREF_PV(pMixedCtx);
13518	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13519	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13520	Assert(pVCpu->hm.s.fUsingDebugLoop);
13521	#endif
13522
13523	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13524	hmR0VmxCheckExitDueToEventDelivery(). */
13525	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13526	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13527	AssertRCReturn(rc, rc);
13528	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13529
13530	#ifdef DEBUG_ramshankar
13531	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13532	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13533	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13534	#endif
13535
13536	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13537	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13538	return VINF_SUCCESS;
13539	}
13540
13541
13542	/**
13543	* VM-exit exception handler for \#PF (Page-fault exception).
13544	*/
13545	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13546	{
13547	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13548	PVM pVM = pVCpu->CTX_SUFF(pVM);
13549	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13550	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13551	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13552	AssertRCReturn(rc, rc);
13553
13554	if (!pVM->hm.s.fNestedPaging)
13555	{ /* likely */ }
13556	else
13557	{
13558	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13559	Assert(pVCpu->hm.s.fUsingDebugLoop);
13560	#endif
13561	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13562	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13563	{
13564	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13565	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13566	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13567	}
13568	else
13569	{
13570	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13571	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13572	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13573	}
13574	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13575	return rc;
13576	}
13577
13578	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13579	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13580	if (pVmxTransient->fVectoringPF)
13581	{
13582	Assert(pVCpu->hm.s.Event.fPending);
13583	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13584	}
13585
13586	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13587	AssertRCReturn(rc, rc);
13588
13589	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13590	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13591
13592	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13593	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13594	(RTGCPTR)pVmxTransient->uExitQualification);
13595
13596	Log4(("#PF: rc=%Rrc\n", rc));
13597	if (rc == VINF_SUCCESS)
13598	{
13599	#if 0
13600	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13601	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13602	* memory? We don't update the whole state here... */
13603	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13604	\| HM_CHANGED_GUEST_RSP
13605	\| HM_CHANGED_GUEST_RFLAGS
13606	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13607	#else
13608	/*
13609	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13610	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13611	*/
13612	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13613	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13614	#endif
13615	TRPMResetTrap(pVCpu);
13616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13617	return rc;
13618	}
13619
13620	if (rc == VINF_EM_RAW_GUEST_TRAP)
13621	{
13622	if (!pVmxTransient->fVectoringDoublePF)
13623	{
13624	/* It's a guest page fault and needs to be reflected to the guest. */
13625	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13626	TRPMResetTrap(pVCpu);
13627	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13628	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13629	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13630	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13631	}
13632	else
13633	{
13634	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13635	TRPMResetTrap(pVCpu);
13636	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13637	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13638	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13639	}
13640
13641	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13642	return VINF_SUCCESS;
13643	}
13644
13645	TRPMResetTrap(pVCpu);
13646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13647	return rc;
13648	}
13649
13650	/** @} */
13651

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 65136

Download in other formats: