HMVMXR0.cpp@ 49000

Last change on this file since 49000 was 49000, checked in by vboxsync, 12 years ago
VMM: IntrInfo to IntInfo renaming.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 453.3 KB

Line
1	/* $Id: HMVMXR0.cpp 49000 2013-10-09 12:22:39Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#define HMVMX_ALWAYS_SWAP_FPU_STATE
45	#endif
46
47
48	/*******************************************************************************
49	* Defined Constants And Macros *
50	*******************************************************************************/
51	#if defined(RT_ARCH_AMD64)
52	# define HMVMX_IS_64BIT_HOST_MODE() (true)
53	typedef RTHCUINTREG HMVMXHCUINTREG;
54	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55	extern "C" uint32_t g_fVMXIs64bitHost;
56	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57	typedef uint64_t HMVMXHCUINTREG;
58	#else
59	# define HMVMX_IS_64BIT_HOST_MODE() (false)
60	typedef RTHCUINTREG HMVMXHCUINTREG;
61	#endif
62
63	/** Use the function table. */
64	#define HMVMX_USE_FUNCTION_TABLE
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	/** @} */
127
128	/** @name
129	* States of the VMCS.
130	*
131	* This does not reflect all possible VMCS states but currently only those
132	* needed for maintaining the VMCS consistently even when thread-context hooks
133	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
134	*/
135	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
136	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
137	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
138	/** @} */
139
140	/**
141	* Exception bitmap mask for real-mode guests (real-on-v86).
142	*
143	* We need to intercept all exceptions manually (except #PF). #NM is also
144	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
145	* even in real-mode if we have Nested Paging support.
146	*/
147	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
148	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
149	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
150	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
151	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
153	\| RT_BIT(X86_XCPT_XF))
154
155	/**
156	* Exception bitmap mask for all contributory exceptions.
157	*
158	* Page fault is deliberately excluded here as it's conditional as to whether
159	* it's contributory or benign. Page faults are handled separately.
160	*/
161	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
162	\| RT_BIT(X86_XCPT_DE))
163
164	/** Maximum VM-instruction error number. */
165	#define HMVMX_INSTR_ERROR_MAX 28
166
167	/** Profiling macro. */
168	#ifdef HM_PROFILE_EXIT_DISPATCH
169	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171	#else
172	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174	#endif
175
176	/** Assert that preemption is disabled or covered by thread-context hooks. */
177	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
178	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180	/** Assert that we haven't migrated CPUs when thread-context hooks are not
181	* used. */
182	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184	("Illegal migration! Entered on CPU %u Current %u\n", \
185	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187	/** Helper macro for VM-exit handlers called unexpectedly. */
188	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189	do { \
190	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191	return VERR_VMX_UNEXPECTED_EXIT; \
192	} while (0)
193
194
195	/*******************************************************************************
196	* Structures and Typedefs *
197	*******************************************************************************/
198	/**
199	* VMX transient state.
200	*
201	* A state structure for holding miscellaneous information across
202	* VMX non-root operation and restored after the transition.
203	*/
204	typedef struct VMXTRANSIENT
205	{
206	/** The host's rflags/eflags. */
207	RTCCUINTREG uEflags;
208	#if HC_ARCH_BITS == 32
209	uint32_t u32Alignment0;
210	#endif
211	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
212	uint64_t u64LStarMsr;
213	/** The guest's TPR value used for TPR shadowing. */
214	uint8_t u8GuestTpr;
215	/** Alignment. */
216	uint8_t abAlignment0[7];
217
218	/** The basic VM-exit reason. */
219	uint16_t uExitReason;
220	/** Alignment. */
221	uint16_t u16Alignment0;
222	/** The VM-exit interruption error code. */
223	uint32_t uExitIntErrorCode;
224	/** The VM-exit exit qualification. */
225	uint64_t uExitQualification;
226
227	/** The VM-exit interruption-information field. */
228	uint32_t uExitIntInfo;
229	/** The VM-exit instruction-length field. */
230	uint32_t cbInstr;
231	/** The VM-exit instruction-information field. */
232	union
233	{
234	/** Plain unsigned int representation. */
235	uint32_t u;
236	/** INS and OUTS information. */
237	struct
238	{
239	uint32_t u6Reserved0 : 6;
240	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241	uint32_t u3AddrSize : 3;
242	uint32_t u5Reserved1 : 5;
243	/** The segment register (X86_SREG_XXX). */
244	uint32_t iSegReg : 3;
245	uint32_t uReserved2 : 14;
246	} StrIo;
247	} ExitInstrInfo;
248	/** Whether the VM-entry failed or not. */
249	bool fVMEntryFailed;
250	/** Alignment. */
251	uint8_t abAlignment1[3];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest FPU was active at the time of VM-exit. */
269	bool fWasGuestFPUStateActive;
270	/** Whether the guest debug state was active at the time of VM-exit. */
271	bool fWasGuestDebugStateActive;
272	/** Whether the hyper debug state was active at the time of VM-exit. */
273	bool fWasHyperDebugStateActive;
274	/** Whether TSC-offsetting should be setup before VM-entry. */
275	bool fUpdateTscOffsettingAndPreemptTimer;
276	/** Whether the VM-exit was caused by a page-fault during delivery of a
277	* contributory exception or a page-fault. */
278	bool fVectoringPF;
279	} VMXTRANSIENT;
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
284	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
285	/** Pointer to VMX transient state. */
286	typedef VMXTRANSIENT *PVMXTRANSIENT;
287
288
289	/**
290	* MSR-bitmap read permissions.
291	*/
292	typedef enum VMXMSREXITREAD
293	{
294	/** Reading this MSR causes a VM-exit. */
295	VMXMSREXIT_INTERCEPT_READ = 0xb,
296	/** Reading this MSR does not cause a VM-exit. */
297	VMXMSREXIT_PASSTHRU_READ
298	} VMXMSREXITREAD;
299
300	/**
301	* MSR-bitmap write permissions.
302	*/
303	typedef enum VMXMSREXITWRITE
304	{
305	/** Writing to this MSR causes a VM-exit. */
306	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
307	/** Writing to this MSR does not cause a VM-exit. */
308	VMXMSREXIT_PASSTHRU_WRITE
309	} VMXMSREXITWRITE;
310
311	/**
312	* VMX VM-exit handler.
313	*
314	* @returns VBox status code.
315	* @param pVCpu Pointer to the VMCPU.
316	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
317	* out-of-sync. Make sure to update the required
318	* fields before using them.
319	* @param pVmxTransient Pointer to the VMX-transient structure.
320	*/
321	#ifndef HMVMX_USE_FUNCTION_TABLE
322	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
323	#else
324	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
325	/** Pointer to VM-exit handler. */
326	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
327	#endif
328
329
330	/*******************************************************************************
331	* Internal Functions *
332	*******************************************************************************/
333	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
334	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
335	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
336	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
337	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
338	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
339	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
340	#endif
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
343	# define HMVMX_EXIT_DECL static int
344	#else
345	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
346	#endif
347
348	/** @name VM-exit handlers.
349	* @{
350	*/
351	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
352	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
353	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
354	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
355	static FNVMXEXITHANDLER hmR0VmxExitSipi;
356	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
357	static FNVMXEXITHANDLER hmR0VmxExitSmi;
358	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
359	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
360	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
361	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
362	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
363	static FNVMXEXITHANDLER hmR0VmxExitHlt;
364	static FNVMXEXITHANDLER hmR0VmxExitInvd;
365	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
366	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
367	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
368	static FNVMXEXITHANDLER hmR0VmxExitRsm;
369	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
370	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
371	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
372	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
373	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
374	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
375	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
376	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
377	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
378	static FNVMXEXITHANDLER hmR0VmxExitMwait;
379	static FNVMXEXITHANDLER hmR0VmxExitMtf;
380	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
381	static FNVMXEXITHANDLER hmR0VmxExitPause;
382	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
383	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
384	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
385	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
386	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
387	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
388	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
389	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
390	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
391	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
392	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
393	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
394	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
395	/** @} */
396
397	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
398	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
399	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
400	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
405
406	/*******************************************************************************
407	* Global Variables *
408	*******************************************************************************/
409	#ifdef HMVMX_USE_FUNCTION_TABLE
410
411	/**
412	* VMX_EXIT dispatch table.
413	*/
414	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
415	{
416	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
417	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
418	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
419	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
420	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
421	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
422	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
423	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
424	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
425	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
426	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
427	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
428	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
429	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
430	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
431	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
432	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
433	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
434	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
435	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
436	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
437	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
438	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
439	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
440	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
441	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
442	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
443	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
444	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
445	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
446	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
447	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
448	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
449	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
450	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
451	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
452	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
453	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
454	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
455	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
456	/* 40 UNDEFINED */ hmR0VmxExitPause,
457	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
458	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
459	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
460	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
461	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
462	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
463	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
464	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
465	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
466	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
467	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
468	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
469	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
470	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
471	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
472	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
473	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
474	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
475	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
476	};
477	#endif /* HMVMX_USE_FUNCTION_TABLE */
478
479	#ifdef VBOX_STRICT
480	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
481	{
482	/* 0 */ "(Not Used)",
483	/* 1 */ "VMCALL executed in VMX root operation.",
484	/* 2 */ "VMCLEAR with invalid physical address.",
485	/* 3 */ "VMCLEAR with VMXON pointer.",
486	/* 4 */ "VMLAUNCH with non-clear VMCS.",
487	/* 5 */ "VMRESUME with non-launched VMCS.",
488	/* 6 */ "VMRESUME after VMXOFF",
489	/* 7 */ "VM entry with invalid control fields.",
490	/* 8 */ "VM entry with invalid host state fields.",
491	/* 9 */ "VMPTRLD with invalid physical address.",
492	/* 10 */ "VMPTRLD with VMXON pointer.",
493	/* 11 */ "VMPTRLD with incorrect revision identifier.",
494	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
495	/* 13 */ "VMWRITE to read-only VMCS component.",
496	/* 14 */ "(Not Used)",
497	/* 15 */ "VMXON executed in VMX root operation.",
498	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
499	/* 17 */ "VM entry with non-launched executing VMCS.",
500	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
501	/* 19 */ "VMCALL with non-clear VMCS.",
502	/* 20 */ "VMCALL with invalid VM-exit control fields.",
503	/* 21 */ "(Not Used)",
504	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
505	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
506	/* 24 */ "VMCALL with invalid SMM-monitor features.",
507	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
508	/* 26 */ "VM entry with events blocked by MOV SS.",
509	/* 27 */ "(Not Used)",
510	/* 28 */ "Invalid operand to INVEPT/INVVPID."
511	};
512	#endif /* VBOX_STRICT */
513
514
515
516	/**
517	* Updates the VM's last error record. If there was a VMX instruction error,
518	* reads the error data from the VMCS and updates VCPU's last error record as
519	* well.
520	*
521	* @param pVM Pointer to the VM.
522	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
523	* VERR_VMX_UNABLE_TO_START_VM or
524	* VERR_VMX_INVALID_VMCS_FIELD).
525	* @param rc The error code.
526	*/
527	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
528	{
529	AssertPtr(pVM);
530	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
531	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
532	{
533	AssertPtrReturnVoid(pVCpu);
534	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
535	}
536	pVM->hm.s.lLastError = rc;
537	}
538
539
540	/**
541	* Reads the VM-entry interruption-information field from the VMCS into the VMX
542	* transient structure.
543	*
544	* @returns VBox status code.
545	* @param pVmxTransient Pointer to the VMX transient structure.
546	*
547	* @remarks No-long-jump zone!!!
548	*/
549	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
550	{
551	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
552	AssertRCReturn(rc, rc);
553	return VINF_SUCCESS;
554	}
555
556
557	/**
558	* Reads the VM-entry exception error code field from the VMCS into
559	* the VMX transient structure.
560	*
561	* @returns VBox status code.
562	* @param pVmxTransient Pointer to the VMX transient structure.
563	*
564	* @remarks No-long-jump zone!!!
565	*/
566	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
567	{
568	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
569	AssertRCReturn(rc, rc);
570	return VINF_SUCCESS;
571	}
572
573
574	/**
575	* Reads the VM-entry exception error code field from the VMCS into
576	* the VMX transient structure.
577	*
578	* @returns VBox status code.
579	* @param pVCpu Pointer to the VMCPU.
580	* @param pVmxTransient Pointer to the VMX transient structure.
581	*
582	* @remarks No-long-jump zone!!!
583	*/
584	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
585	{
586	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
587	AssertRCReturn(rc, rc);
588	return VINF_SUCCESS;
589	}
590
591
592	/**
593	* Reads the VM-exit interruption-information field from the VMCS into the VMX
594	* transient structure.
595	*
596	* @returns VBox status code.
597	* @param pVCpu Pointer to the VMCPU.
598	* @param pVmxTransient Pointer to the VMX transient structure.
599	*/
600	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
601	{
602	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
603	{
604	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
605	AssertRCReturn(rc, rc);
606	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
607	}
608	return VINF_SUCCESS;
609	}
610
611
612	/**
613	* Reads the VM-exit interruption error code from the VMCS into the VMX
614	* transient structure.
615	*
616	* @returns VBox status code.
617	* @param pVCpu Pointer to the VMCPU.
618	* @param pVmxTransient Pointer to the VMX transient structure.
619	*/
620	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
621	{
622	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
623	{
624	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
625	AssertRCReturn(rc, rc);
626	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
627	}
628	return VINF_SUCCESS;
629	}
630
631
632	/**
633	* Reads the VM-exit instruction length field from the VMCS into the VMX
634	* transient structure.
635	*
636	* @returns VBox status code.
637	* @param pVCpu Pointer to the VMCPU.
638	* @param pVmxTransient Pointer to the VMX transient structure.
639	*/
640	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
641	{
642	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
643	{
644	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
645	AssertRCReturn(rc, rc);
646	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
647	}
648	return VINF_SUCCESS;
649	}
650
651
652	/**
653	* Reads the VM-exit instruction-information field from the VMCS into
654	* the VMX transient structure.
655	*
656	* @returns VBox status code.
657	* @param pVCpu The cross context per CPU structure.
658	* @param pVmxTransient Pointer to the VMX transient structure.
659	*/
660	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
661	{
662	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
665	AssertRCReturn(rc, rc);
666	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
667	}
668	return VINF_SUCCESS;
669	}
670
671
672	/**
673	* Reads the exit qualification from the VMCS into the VMX transient structure.
674	*
675	* @returns VBox status code.
676	* @param pVCpu Pointer to the VMCPU.
677	* @param pVmxTransient Pointer to the VMX transient structure.
678	*/
679	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
680	{
681	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
682	{
683	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
684	AssertRCReturn(rc, rc);
685	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
686	}
687	return VINF_SUCCESS;
688	}
689
690
691	/**
692	* Reads the IDT-vectoring information field from the VMCS into the VMX
693	* transient structure.
694	*
695	* @returns VBox status code.
696	* @param pVmxTransient Pointer to the VMX transient structure.
697	*
698	* @remarks No-long-jump zone!!!
699	*/
700	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
701	{
702	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
703	{
704	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
705	AssertRCReturn(rc, rc);
706	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
707	}
708	return VINF_SUCCESS;
709	}
710
711
712	/**
713	* Reads the IDT-vectoring error code from the VMCS into the VMX
714	* transient structure.
715	*
716	* @returns VBox status code.
717	* @param pVmxTransient Pointer to the VMX transient structure.
718	*/
719	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
720	{
721	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
722	{
723	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
724	AssertRCReturn(rc, rc);
725	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
726	}
727	return VINF_SUCCESS;
728	}
729
730
731	/**
732	* Enters VMX root mode operation on the current CPU.
733	*
734	* @returns VBox status code.
735	* @param pVM Pointer to the VM (optional, can be NULL, after
736	* a resume).
737	* @param HCPhysCpuPage Physical address of the VMXON region.
738	* @param pvCpuPage Pointer to the VMXON region.
739	*/
740	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
741	{
742	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
743	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
744	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
745
746	if (pVM)
747	{
748	/* Write the VMCS revision dword to the VMXON region. */
749	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
750	}
751
752	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
753	RTCCUINTREG uEflags = ASMIntDisableFlags();
754
755	/* Enable the VMX bit in CR4 if necessary. */
756	RTCCUINTREG uCr4 = ASMGetCR4();
757	if (!(uCr4 & X86_CR4_VMXE))
758	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
759
760	/* Enter VMX root mode. */
761	int rc = VMXEnable(HCPhysCpuPage);
762	if (RT_FAILURE(rc))
763	ASMSetCR4(uCr4);
764
765	/* Restore interrupts. */
766	ASMSetFlags(uEflags);
767	return rc;
768	}
769
770
771	/**
772	* Exits VMX root mode operation on the current CPU.
773	*
774	* @returns VBox status code.
775	*/
776	static int hmR0VmxLeaveRootMode(void)
777	{
778	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
779
780	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
781	RTCCUINTREG uEflags = ASMIntDisableFlags();
782
783	/* If we're for some reason not in VMX root mode, then don't leave it. */
784	RTCCUINTREG uHostCR4 = ASMGetCR4();
785
786	int rc;
787	if (uHostCR4 & X86_CR4_VMXE)
788	{
789	/* Exit VMX root mode and clear the VMX bit in CR4. */
790	VMXDisable();
791	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
792	rc = VINF_SUCCESS;
793	}
794	else
795	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
796
797	/* Restore interrupts. */
798	ASMSetFlags(uEflags);
799	return rc;
800	}
801
802
803	/**
804	* Allocates and maps one physically contiguous page. The allocated page is
805	* zero'd out. (Used by various VT-x structures).
806	*
807	* @returns IPRT status code.
808	* @param pMemObj Pointer to the ring-0 memory object.
809	* @param ppVirt Where to store the virtual address of the
810	* allocation.
811	* @param pPhys Where to store the physical address of the
812	* allocation.
813	*/
814	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
815	{
816	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
817	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
818	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
819
820	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
821	if (RT_FAILURE(rc))
822	return rc;
823	ppVirt = RTR0MemObjAddress(pMemObj);
824	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
825	ASMMemZero32(*ppVirt, PAGE_SIZE);
826	return VINF_SUCCESS;
827	}
828
829
830	/**
831	* Frees and unmaps an allocated physical page.
832	*
833	* @param pMemObj Pointer to the ring-0 memory object.
834	* @param ppVirt Where to re-initialize the virtual address of
835	* allocation as 0.
836	* @param pHCPhys Where to re-initialize the physical address of the
837	* allocation as 0.
838	*/
839	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
840	{
841	AssertPtr(pMemObj);
842	AssertPtr(ppVirt);
843	AssertPtr(pHCPhys);
844	if (*pMemObj != NIL_RTR0MEMOBJ)
845	{
846	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
847	AssertRC(rc);
848	*pMemObj = NIL_RTR0MEMOBJ;
849	*ppVirt = 0;
850	*pHCPhys = 0;
851	}
852	}
853
854
855	/**
856	* Worker function to free VT-x related structures.
857	*
858	* @returns IPRT status code.
859	* @param pVM Pointer to the VM.
860	*/
861	static void hmR0VmxStructsFree(PVM pVM)
862	{
863	for (VMCPUID i = 0; i < pVM->cCpus; i++)
864	{
865	PVMCPU pVCpu = &pVM->aCpus[i];
866	AssertPtr(pVCpu);
867
868	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
869	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
871	#endif
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
922	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
923	#endif
924	}
925	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
926	#undef VMXLOCAL_INIT_VM_MEMOBJ
927
928	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
929	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
930	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
931	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
932
933	/*
934	* Allocate all the VT-x structures.
935	*/
936	int rc = VINF_SUCCESS;
937	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
938	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
939	if (RT_FAILURE(rc))
940	goto cleanup;
941	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
942	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
943	#endif
944
945	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
946	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
947	{
948	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
949	&pVM->hm.s.vmx.HCPhysApicAccess);
950	if (RT_FAILURE(rc))
951	goto cleanup;
952	}
953
954	/*
955	* Initialize per-VCPU VT-x structures.
956	*/
957	for (VMCPUID i = 0; i < pVM->cCpus; i++)
958	{
959	PVMCPU pVCpu = &pVM->aCpus[i];
960	AssertPtr(pVCpu);
961
962	/* Allocate the VM control structure (VMCS). */
963	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
964	if (RT_FAILURE(rc))
965	goto cleanup;
966
967	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
968	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
969	{
970	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
971	&pVCpu->hm.s.vmx.HCPhysVirtApic);
972	if (RT_FAILURE(rc))
973	goto cleanup;
974	}
975
976	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
977	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
978	{
979	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
980	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
981	if (RT_FAILURE(rc))
982	goto cleanup;
983	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
984	}
985
986	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
987	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
988	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
989	if (RT_FAILURE(rc))
990	goto cleanup;
991
992	/* Allocate the VM-exit MSR-load page for the host MSRs. */
993	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
994	if (RT_FAILURE(rc))
995	goto cleanup;
996	#endif
997	}
998
999	return VINF_SUCCESS;
1000
1001	cleanup:
1002	hmR0VmxStructsFree(pVM);
1003	return rc;
1004	}
1005
1006
1007	/**
1008	* Does global VT-x initialization (called during module initialization).
1009	*
1010	* @returns VBox status code.
1011	*/
1012	VMMR0DECL(int) VMXR0GlobalInit(void)
1013	{
1014	#ifdef HMVMX_USE_FUNCTION_TABLE
1015	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1016	# ifdef VBOX_STRICT
1017	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1018	Assert(g_apfnVMExitHandlers[i]);
1019	# endif
1020	#endif
1021	return VINF_SUCCESS;
1022	}
1023
1024
1025	/**
1026	* Does global VT-x termination (called during module termination).
1027	*/
1028	VMMR0DECL(void) VMXR0GlobalTerm()
1029	{
1030	/* Nothing to do currently. */
1031	}
1032
1033
1034	/**
1035	* Sets up and activates VT-x on the current CPU.
1036	*
1037	* @returns VBox status code.
1038	* @param pCpu Pointer to the global CPU info struct.
1039	* @param pVM Pointer to the VM (can be NULL after a host resume
1040	* operation).
1041	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1042	* fEnabledByHost is true).
1043	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1044	* @a fEnabledByHost is true).
1045	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1046	* enable VT-x on the host.
1047	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1048	*/
1049	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1050	void *pvMsrs)
1051	{
1052	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1053	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1054	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1055
1056	/* Enable VT-x if it's not already enabled by the host. */
1057	if (!fEnabledByHost)
1058	{
1059	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1060	if (RT_FAILURE(rc))
1061	return rc;
1062	}
1063
1064	/*
1065	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1066	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1067	*/
1068	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1069	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1070	{
1071	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1072	pCpu->fFlushAsidBeforeUse = false;
1073	}
1074	else
1075	pCpu->fFlushAsidBeforeUse = true;
1076
1077	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1078	++pCpu->cTlbFlushes;
1079
1080	return VINF_SUCCESS;
1081	}
1082
1083
1084	/**
1085	* Deactivates VT-x on the current CPU.
1086	*
1087	* @returns VBox status code.
1088	* @param pCpu Pointer to the global CPU info struct.
1089	* @param pvCpuPage Pointer to the VMXON region.
1090	* @param HCPhysCpuPage Physical address of the VMXON region.
1091	*
1092	* @remarks This function should never be called when SUPR0EnableVTx() or
1093	* similar was used to enable VT-x on the host.
1094	*/
1095	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1096	{
1097	NOREF(pCpu);
1098	NOREF(pvCpuPage);
1099	NOREF(HCPhysCpuPage);
1100
1101	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1102	return hmR0VmxLeaveRootMode();
1103	}
1104
1105
1106	/**
1107	* Sets the permission bits for the specified MSR in the MSR bitmap.
1108	*
1109	* @param pVCpu Pointer to the VMCPU.
1110	* @param uMSR The MSR value.
1111	* @param enmRead Whether reading this MSR causes a VM-exit.
1112	* @param enmWrite Whether writing this MSR causes a VM-exit.
1113	*/
1114	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1115	{
1116	int32_t iBit;
1117	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1118
1119	/*
1120	* Layout:
1121	* 0x000 - 0x3ff - Low MSR read bits
1122	* 0x400 - 0x7ff - High MSR read bits
1123	* 0x800 - 0xbff - Low MSR write bits
1124	* 0xc00 - 0xfff - High MSR write bits
1125	*/
1126	if (uMsr <= 0x00001FFF)
1127	iBit = uMsr;
1128	else if ( uMsr >= 0xC0000000
1129	&& uMsr <= 0xC0001FFF)
1130	{
1131	iBit = (uMsr - 0xC0000000);
1132	pbMsrBitmap += 0x400;
1133	}
1134	else
1135	{
1136	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1137	return;
1138	}
1139
1140	Assert(iBit <= 0x1fff);
1141	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1142	ASMBitSet(pbMsrBitmap, iBit);
1143	else
1144	ASMBitClear(pbMsrBitmap, iBit);
1145
1146	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1147	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1148	else
1149	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1150	}
1151
1152
1153	/**
1154	* Flushes the TLB using EPT.
1155	*
1156	* @returns VBox status code.
1157	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1158	* enmFlush).
1159	* @param enmFlush Type of flush.
1160	*
1161	* @remarks Caller is responsible for making sure this function is called only
1162	* when NestedPaging is supported and providing @a enmFlush that is
1163	* supported by the CPU.
1164	*/
1165	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1166	{
1167	uint64_t au64Descriptor[2];
1168	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1169	au64Descriptor[0] = 0;
1170	else
1171	{
1172	Assert(pVCpu);
1173	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1174	}
1175	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1176
1177	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1178	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1179	rc));
1180	if ( RT_SUCCESS(rc)
1181	&& pVCpu)
1182	{
1183	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1184	}
1185	}
1186
1187
1188	/**
1189	* Flushes the TLB using VPID.
1190	*
1191	* @returns VBox status code.
1192	* @param pVM Pointer to the VM.
1193	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1194	* enmFlush).
1195	* @param enmFlush Type of flush.
1196	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1197	* on @a enmFlush).
1198	*/
1199	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1200	{
1201	AssertPtr(pVM);
1202	Assert(pVM->hm.s.vmx.fVpid);
1203
1204	uint64_t au64Descriptor[2];
1205	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1206	{
1207	au64Descriptor[0] = 0;
1208	au64Descriptor[1] = 0;
1209	}
1210	else
1211	{
1212	AssertPtr(pVCpu);
1213	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1214	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1215	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1216	au64Descriptor[1] = GCPtr;
1217	}
1218
1219	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1220	AssertMsg(rc == VINF_SUCCESS,
1221	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1222	if ( RT_SUCCESS(rc)
1223	&& pVCpu)
1224	{
1225	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1226	}
1227	}
1228
1229
1230	/**
1231	* Invalidates a guest page by guest virtual address. Only relevant for
1232	* EPT/VPID, otherwise there is nothing really to invalidate.
1233	*
1234	* @returns VBox status code.
1235	* @param pVM Pointer to the VM.
1236	* @param pVCpu Pointer to the VMCPU.
1237	* @param GCVirt Guest virtual address of the page to invalidate.
1238	*/
1239	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1240	{
1241	AssertPtr(pVM);
1242	AssertPtr(pVCpu);
1243	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1244
1245	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1246	if (!fFlushPending)
1247	{
1248	/*
1249	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1250	* See @bugref{6043} and @bugref{6177}.
1251	*
1252	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1253	* function maybe called in a loop with individual addresses.
1254	*/
1255	if (pVM->hm.s.vmx.fVpid)
1256	{
1257	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1258	{
1259	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1260	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1261	}
1262	else
1263	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1264	}
1265	else if (pVM->hm.s.fNestedPaging)
1266	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1267	}
1268
1269	return VINF_SUCCESS;
1270	}
1271
1272
1273	/**
1274	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1275	* otherwise there is nothing really to invalidate.
1276	*
1277	* @returns VBox status code.
1278	* @param pVM Pointer to the VM.
1279	* @param pVCpu Pointer to the VMCPU.
1280	* @param GCPhys Guest physical address of the page to invalidate.
1281	*/
1282	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1283	{
1284	LogFlowFunc(("%RGp\n", GCPhys));
1285
1286	/*
1287	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1288	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1289	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1290	*/
1291	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1292	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1299	* case where neither EPT nor VPID is supported by the CPU.
1300	*
1301	* @param pVM Pointer to the VM.
1302	* @param pVCpu Pointer to the VMCPU.
1303	* @param pCpu Pointer to the global HM struct.
1304	*
1305	* @remarks Called with interrupts disabled.
1306	*/
1307	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1308	{
1309	AssertPtr(pVCpu);
1310	AssertPtr(pCpu);
1311	NOREF(pVM);
1312
1313	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1314	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1315
1316	pVCpu->hm.s.TlbShootdown.cPages = 0;
1317	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1318	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1319	pVCpu->hm.s.fForceTLBFlush = false;
1320	return;
1321	}
1322
1323
1324	/**
1325	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1326	*
1327	* @param pVM Pointer to the VM.
1328	* @param pVCpu Pointer to the VMCPU.
1329	* @param pCpu Pointer to the global HM CPU struct.
1330	* @remarks All references to "ASID" in this function pertains to "VPID" in
1331	* Intel's nomenclature. The reason is, to avoid confusion in compare
1332	* statements since the host-CPU copies are named "ASID".
1333	*
1334	* @remarks Called with interrupts disabled.
1335	*/
1336	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1337	{
1338	#ifdef VBOX_WITH_STATISTICS
1339	bool fTlbFlushed = false;
1340	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1341	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1342	if (!fTlbFlushed) \
1343	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1344	} while (0)
1345	#else
1346	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1347	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1348	#endif
1349
1350	AssertPtr(pVM);
1351	AssertPtr(pCpu);
1352	AssertPtr(pVCpu);
1353	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1354	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1355	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1356
1357
1358	/*
1359	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1360	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1361	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1362	*/
1363	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1364	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1365	{
1366	++pCpu->uCurrentAsid;
1367	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1368	{
1369	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1370	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1371	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1372	}
1373
1374	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1375	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1376	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1377
1378	/*
1379	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1380	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1381	*/
1382	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1383	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1384	HMVMX_SET_TAGGED_TLB_FLUSHED();
1385	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1386	}
1387
1388	/* Check for explicit TLB shootdowns. */
1389	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1390	{
1391	/*
1392	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1393	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1394	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1395	* but not guest-physical mappings.
1396	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1397	*/
1398	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1399	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1400	HMVMX_SET_TAGGED_TLB_FLUSHED();
1401	}
1402
1403	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1404	* not be executed. See hmQueueInvlPage() where it is commented
1405	* out. Support individual entry flushing someday. */
1406	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1407	{
1408	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1409
1410	/*
1411	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1412	* as supported by the CPU.
1413	*/
1414	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1415	{
1416	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1417	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1418	}
1419	else
1420	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1421
1422	HMVMX_SET_TAGGED_TLB_FLUSHED();
1423	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1424	}
1425
1426	pVCpu->hm.s.TlbShootdown.cPages = 0;
1427	pVCpu->hm.s.fForceTLBFlush = false;
1428
1429	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1430
1431	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1432	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1433	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1434	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1435	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1436	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1437	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1438	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1439
1440	/* Update VMCS with the VPID. */
1441	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1442	AssertRC(rc);
1443
1444	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1445	}
1446
1447
1448	/**
1449	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1450	*
1451	* @returns VBox status code.
1452	* @param pVM Pointer to the VM.
1453	* @param pVCpu Pointer to the VMCPU.
1454	* @param pCpu Pointer to the global HM CPU struct.
1455	*
1456	* @remarks Called with interrupts disabled.
1457	*/
1458	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1459	{
1460	AssertPtr(pVM);
1461	AssertPtr(pVCpu);
1462	AssertPtr(pCpu);
1463	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1464	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1465
1466	/*
1467	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1468	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1469	*/
1470	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1471	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1472	{
1473	pVCpu->hm.s.fForceTLBFlush = true;
1474	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1475	}
1476
1477	/* Check for explicit TLB shootdown flushes. */
1478	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1479	{
1480	pVCpu->hm.s.fForceTLBFlush = true;
1481	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1482	}
1483
1484	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1485	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1486
1487	if (pVCpu->hm.s.fForceTLBFlush)
1488	{
1489	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1490	pVCpu->hm.s.fForceTLBFlush = false;
1491	}
1492	else
1493	{
1494	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1495	* not be executed. See hmQueueInvlPage() where it is commented
1496	* out. Support individual entry flushing someday. */
1497	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1498	{
1499	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1500	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1501	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1502	}
1503	else
1504	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1505	}
1506
1507	pVCpu->hm.s.TlbShootdown.cPages = 0;
1508	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1509	}
1510
1511
1512	/**
1513	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1514	*
1515	* @returns VBox status code.
1516	* @param pVM Pointer to the VM.
1517	* @param pVCpu Pointer to the VMCPU.
1518	* @param pCpu Pointer to the global HM CPU struct.
1519	*
1520	* @remarks Called with interrupts disabled.
1521	*/
1522	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1523	{
1524	AssertPtr(pVM);
1525	AssertPtr(pVCpu);
1526	AssertPtr(pCpu);
1527	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1528	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1529
1530	/*
1531	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1532	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1533	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1534	*/
1535	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1536	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1537	{
1538	pVCpu->hm.s.fForceTLBFlush = true;
1539	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1540	}
1541
1542	/* Check for explicit TLB shootdown flushes. */
1543	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1544	{
1545	/*
1546	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1547	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1548	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1549	*/
1550	pVCpu->hm.s.fForceTLBFlush = true;
1551	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1552	}
1553
1554	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1555	if (pVCpu->hm.s.fForceTLBFlush)
1556	{
1557	++pCpu->uCurrentAsid;
1558	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1559	{
1560	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1561	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1562	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1563	}
1564
1565	pVCpu->hm.s.fForceTLBFlush = false;
1566	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1567	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1568	if (pCpu->fFlushAsidBeforeUse)
1569	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1570	}
1571	else
1572	{
1573	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1574	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1575	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1576	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1577
1578	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1579	* not be executed. See hmQueueInvlPage() where it is commented
1580	* out. Support individual entry flushing someday. */
1581	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1582	{
1583	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1584	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1585	{
1586	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1587	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1588	}
1589	else
1590	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1591	}
1592	else
1593	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1594	}
1595
1596	pVCpu->hm.s.TlbShootdown.cPages = 0;
1597	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1598
1599	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1600	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1601	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1602	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1603	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1604	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1605
1606	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1607	AssertRC(rc);
1608	}
1609
1610
1611	/**
1612	* Flushes the guest TLB entry based on CPU capabilities.
1613	*
1614	* @param pVCpu Pointer to the VMCPU.
1615	* @param pCpu Pointer to the global HM CPU struct.
1616	*/
1617	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1618	{
1619	PVM pVM = pVCpu->CTX_SUFF(pVM);
1620	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1621	{
1622	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1623	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1624	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1625	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1626	default:
1627	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1628	break;
1629	}
1630	}
1631
1632
1633	/**
1634	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1635	* TLB entries from the host TLB before VM-entry.
1636	*
1637	* @returns VBox status code.
1638	* @param pVM Pointer to the VM.
1639	*/
1640	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1641	{
1642	/*
1643	* Determine optimal flush type for Nested Paging.
1644	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1645	* guest execution (see hmR3InitFinalizeR0()).
1646	*/
1647	if (pVM->hm.s.fNestedPaging)
1648	{
1649	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1650	{
1651	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1652	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1653	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1654	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1655	else
1656	{
1657	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1658	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1659	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1660	}
1661
1662	/* Make sure the write-back cacheable memory type for EPT is supported. */
1663	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1664	{
1665	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1666	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1667	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1668	}
1669	}
1670	else
1671	{
1672	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1673	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1674	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1675	}
1676	}
1677
1678	/*
1679	* Determine optimal flush type for VPID.
1680	*/
1681	if (pVM->hm.s.vmx.fVpid)
1682	{
1683	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1684	{
1685	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1686	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1687	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1688	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1689	else
1690	{
1691	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1692	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1693	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1694	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1695	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1696	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1697	pVM->hm.s.vmx.fVpid = false;
1698	}
1699	}
1700	else
1701	{
1702	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1703	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1704	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1705	pVM->hm.s.vmx.fVpid = false;
1706	}
1707	}
1708
1709	/*
1710	* Setup the handler for flushing tagged-TLBs.
1711	*/
1712	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1713	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1714	else if (pVM->hm.s.fNestedPaging)
1715	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1716	else if (pVM->hm.s.vmx.fVpid)
1717	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1718	else
1719	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1720	return VINF_SUCCESS;
1721	}
1722
1723
1724	/**
1725	* Sets up pin-based VM-execution controls in the VMCS.
1726	*
1727	* @returns VBox status code.
1728	* @param pVM Pointer to the VM.
1729	* @param pVCpu Pointer to the VMCPU.
1730	*/
1731	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1732	{
1733	AssertPtr(pVM);
1734	AssertPtr(pVCpu);
1735
1736	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1737	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1738
1739	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1740	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1741	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1742
1743	/* Enable the VMX preemption timer. */
1744	if (pVM->hm.s.vmx.fUsePreemptTimer)
1745	{
1746	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1747	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1748	}
1749
1750	if ((val & zap) != val)
1751	{
1752	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1753	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1754	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1755	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1756	}
1757
1758	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1759	AssertRCReturn(rc, rc);
1760
1761	/* Update VCPU with the currently set pin-based VM-execution controls. */
1762	pVCpu->hm.s.vmx.u32PinCtls = val;
1763	return rc;
1764	}
1765
1766
1767	/**
1768	* Sets up processor-based VM-execution controls in the VMCS.
1769	*
1770	* @returns VBox status code.
1771	* @param pVM Pointer to the VM.
1772	* @param pVMCPU Pointer to the VMCPU.
1773	*/
1774	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1775	{
1776	AssertPtr(pVM);
1777	AssertPtr(pVCpu);
1778
1779	int rc = VERR_INTERNAL_ERROR_5;
1780	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1781	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1782
1783	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1784	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1785	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1786	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1787	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1788	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1789	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1790
1791	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1792	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1793	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1794	{
1795	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1796	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1797	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1798	}
1799
1800	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1801	if (!pVM->hm.s.fNestedPaging)
1802	{
1803	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1804	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1805	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1806	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1807	}
1808
1809	/* Use TPR shadowing if supported by the CPU. */
1810	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1811	{
1812	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1813	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1814	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1815	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1816	AssertRCReturn(rc, rc);
1817
1818	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1819	/* CR8 writes causes a VM-exit based on TPR threshold. */
1820	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1821	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1822	}
1823	else
1824	{
1825	/*
1826	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
1827	* Set this control only for 64-bit guests.
1828	*/
1829	if (pVM->hm.s.fAllow64BitGuests)
1830	{
1831	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1832	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1833	}
1834	}
1835
1836	/* Use MSR-bitmaps if supported by the CPU. */
1837	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1838	{
1839	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1840
1841	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1842	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1843	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1844	AssertRCReturn(rc, rc);
1845
1846	/*
1847	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1848	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1849	*/
1850	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1851	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1852	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1853	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1854	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1855	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1856	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1857	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1858	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1859	}
1860
1861	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1862	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1863	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1864
1865	if ((val & zap) != val)
1866	{
1867	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1868	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1869	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1870	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1871	}
1872
1873	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1874	AssertRCReturn(rc, rc);
1875
1876	/* Update VCPU with the currently set processor-based VM-execution controls. */
1877	pVCpu->hm.s.vmx.u32ProcCtls = val;
1878
1879	/*
1880	* Secondary processor-based VM-execution controls.
1881	*/
1882	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1883	{
1884	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1885	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1886
1887	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1888	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1889
1890	if (pVM->hm.s.fNestedPaging)
1891	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1892	else
1893	{
1894	/*
1895	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1896	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1897	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1898	*/
1899	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1900	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1901	}
1902
1903	if (pVM->hm.s.vmx.fVpid)
1904	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1905
1906	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1907	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1908
1909	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1910	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1911	* done dynamically. */
1912	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1913	{
1914	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1915	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1916	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1917	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1918	AssertRCReturn(rc, rc);
1919	}
1920
1921	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1922	{
1923	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1924	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1925	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1926	}
1927
1928	if ((val & zap) != val)
1929	{
1930	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1931	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1932	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1933	}
1934
1935	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1936	AssertRCReturn(rc, rc);
1937
1938	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1939	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1940	}
1941	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1942	{
1943	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1944	"available\n"));
1945	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1946	}
1947
1948	return VINF_SUCCESS;
1949	}
1950
1951
1952	/**
1953	* Sets up miscellaneous (everything other than Pin & Processor-based
1954	* VM-execution) control fields in the VMCS.
1955	*
1956	* @returns VBox status code.
1957	* @param pVM Pointer to the VM.
1958	* @param pVCpu Pointer to the VMCPU.
1959	*/
1960	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1961	{
1962	AssertPtr(pVM);
1963	AssertPtr(pVCpu);
1964
1965	int rc = VERR_GENERAL_FAILURE;
1966
1967	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1968	#if 0
1969	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
1970	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1971	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1972
1973	/*
1974	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1975	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1976	* We thus use the exception bitmap to control it rather than use both.
1977	*/
1978	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1979	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1980
1981	/** @todo Explore possibility of using IO-bitmaps. */
1982	/* All IO & IOIO instructions cause VM-exits. */
1983	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1984	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1985
1986	/* Initialize the MSR-bitmap area. */
1987	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1988	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1989	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1990	#endif
1991
1992	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1993	/* Setup MSR autoloading/storing. */
1994	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1995	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1996	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1997	AssertRCReturn(rc, rc);
1998	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1999	AssertRCReturn(rc, rc);
2000
2001	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2002	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2003	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2004	AssertRCReturn(rc, rc);
2005	#endif
2006
2007	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2008	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2009	AssertRCReturn(rc, rc);
2010
2011	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2012	#if 0
2013	/* Setup debug controls */
2014	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2015	AssertRCReturn(rc, rc);
2016	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2017	AssertRCReturn(rc, rc);
2018	#endif
2019
2020	return rc;
2021	}
2022
2023
2024	/**
2025	* Sets up the initial exception bitmap in the VMCS based on static conditions
2026	* (i.e. conditions that cannot ever change after starting the VM).
2027	*
2028	* @returns VBox status code.
2029	* @param pVM Pointer to the VM.
2030	* @param pVCpu Pointer to the VMCPU.
2031	*/
2032	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2033	{
2034	AssertPtr(pVM);
2035	AssertPtr(pVCpu);
2036
2037	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2038
2039	uint32_t u32XcptBitmap = 0;
2040
2041	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2042	if (!pVM->hm.s.fNestedPaging)
2043	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2044
2045	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2046	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2047	AssertRCReturn(rc, rc);
2048	return rc;
2049	}
2050
2051
2052	/**
2053	* Sets up the initial guest-state mask. The guest-state mask is consulted
2054	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2055	* for the nested virtualization case (as it would cause a VM-exit).
2056	*
2057	* @param pVCpu Pointer to the VMCPU.
2058	*/
2059	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2060	{
2061	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2062	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2063	return VINF_SUCCESS;
2064	}
2065
2066
2067	/**
2068	* Does per-VM VT-x initialization.
2069	*
2070	* @returns VBox status code.
2071	* @param pVM Pointer to the VM.
2072	*/
2073	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2074	{
2075	LogFlowFunc(("pVM=%p\n", pVM));
2076
2077	int rc = hmR0VmxStructsAlloc(pVM);
2078	if (RT_FAILURE(rc))
2079	{
2080	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2081	return rc;
2082	}
2083
2084	return VINF_SUCCESS;
2085	}
2086
2087
2088	/**
2089	* Does per-VM VT-x termination.
2090	*
2091	* @returns VBox status code.
2092	* @param pVM Pointer to the VM.
2093	*/
2094	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2095	{
2096	LogFlowFunc(("pVM=%p\n", pVM));
2097
2098	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2099	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2100	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2101	#endif
2102	hmR0VmxStructsFree(pVM);
2103	return VINF_SUCCESS;
2104	}
2105
2106
2107	/**
2108	* Sets up the VM for execution under VT-x.
2109	* This function is only called once per-VM during initialization.
2110	*
2111	* @returns VBox status code.
2112	* @param pVM Pointer to the VM.
2113	*/
2114	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2115	{
2116	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2117	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2118
2119	LogFlowFunc(("pVM=%p\n", pVM));
2120
2121	/*
2122	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2123	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2124	*/
2125	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2126	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2127	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2128	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2129	{
2130	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2131	return VERR_INTERNAL_ERROR;
2132	}
2133
2134	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2135	/*
2136	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2137	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2138	*/
2139	if ( pVM->hm.s.fAllow64BitGuests
2140	&& !HMVMX_IS_64BIT_HOST_MODE())
2141	{
2142	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2143	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2144	}
2145	#endif
2146
2147	/* Initialize these always, see hmR3InitFinalizeR0().*/
2148	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2149	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2150
2151	/* Setup the tagged-TLB flush handlers. */
2152	int rc = hmR0VmxSetupTaggedTlb(pVM);
2153	if (RT_FAILURE(rc))
2154	{
2155	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2156	return rc;
2157	}
2158
2159	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2160	{
2161	PVMCPU pVCpu = &pVM->aCpus[i];
2162	AssertPtr(pVCpu);
2163	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2164
2165	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2166	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2167
2168	/* Set revision dword at the beginning of the VMCS structure. */
2169	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2170
2171	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2172	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2173	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2174	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2175
2176	/* Load this VMCS as the current VMCS. */
2177	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2178	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2179	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2180
2181	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2182	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2183	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2184
2185	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2186	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2187	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2188
2189	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2190	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2191	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2192
2193	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2194	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2195	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2196
2197	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2198	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2199	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2200
2201	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2202	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2203	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2204	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2205	#endif
2206
2207	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2208	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2209	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2210	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2211
2212	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2213
2214	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2215	}
2216
2217	return VINF_SUCCESS;
2218	}
2219
2220
2221	/**
2222	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2223	* the VMCS.
2224	*
2225	* @returns VBox status code.
2226	* @param pVM Pointer to the VM.
2227	* @param pVCpu Pointer to the VMCPU.
2228	*/
2229	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2230	{
2231	RTCCUINTREG uReg = ASMGetCR0();
2232	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2233	AssertRCReturn(rc, rc);
2234
2235	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2236	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2237	if (HMVMX_IS_64BIT_HOST_MODE())
2238	{
2239	uint64_t uRegCR3 = HMR0Get64bitCR3();
2240	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2241	}
2242	else
2243	#endif
2244	{
2245	uReg = ASMGetCR3();
2246	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2247	}
2248	AssertRCReturn(rc, rc);
2249
2250	uReg = ASMGetCR4();
2251	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2252	AssertRCReturn(rc, rc);
2253	return rc;
2254	}
2255
2256
2257	#if HC_ARCH_BITS == 64
2258	/**
2259	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2260	* requirements. See hmR0VmxSaveHostSegmentRegs().
2261	*/
2262	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2263	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2264	{ \
2265	bool fValidSelector = true; \
2266	if ((selValue) & X86_SEL_LDT) \
2267	{ \
2268	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2269	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2270	} \
2271	if (fValidSelector) \
2272	{ \
2273	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2274	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2275	} \
2276	(selValue) = 0; \
2277	}
2278	#endif
2279
2280
2281	/**
2282	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2283	* the host-state area in the VMCS.
2284	*
2285	* @returns VBox status code.
2286	* @param pVM Pointer to the VM.
2287	* @param pVCpu Pointer to the VMCPU.
2288	*/
2289	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2290	{
2291	int rc = VERR_INTERNAL_ERROR_5;
2292
2293	/*
2294	* Host DS, ES, FS and GS segment registers.
2295	*/
2296	#if HC_ARCH_BITS == 64
2297	RTSEL uSelDS = ASMGetDS();
2298	RTSEL uSelES = ASMGetES();
2299	RTSEL uSelFS = ASMGetFS();
2300	RTSEL uSelGS = ASMGetGS();
2301	#else
2302	RTSEL uSelDS = 0;
2303	RTSEL uSelES = 0;
2304	RTSEL uSelFS = 0;
2305	RTSEL uSelGS = 0;
2306	#endif
2307
2308	/* Recalculate which host-state bits need to be manually restored. */
2309	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2310
2311	/*
2312	* Host CS and SS segment registers.
2313	*/
2314	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2315	RTSEL uSelCS;
2316	RTSEL uSelSS;
2317	if (HMVMX_IS_64BIT_HOST_MODE())
2318	{
2319	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2320	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2321	}
2322	else
2323	{
2324	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2325	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2326	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2327	}
2328	#else
2329	RTSEL uSelCS = ASMGetCS();
2330	RTSEL uSelSS = ASMGetSS();
2331	#endif
2332
2333	/*
2334	* Host TR segment register.
2335	*/
2336	RTSEL uSelTR = ASMGetTR();
2337
2338	#if HC_ARCH_BITS == 64
2339	/*
2340	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2341	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2342	*/
2343	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2344	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2345	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2346	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2347	# undef VMXLOCAL_ADJUST_HOST_SEG
2348	#endif
2349
2350	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2351	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2352	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2353	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2354	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2355	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2356	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2357	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2358	Assert(uSelCS);
2359	Assert(uSelTR);
2360
2361	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2362	#if 0
2363	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2364	Assert(uSelSS != 0);
2365	#endif
2366
2367	/* Write these host selector fields into the host-state area in the VMCS. */
2368	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2369	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2370	#if HC_ARCH_BITS == 64
2371	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2372	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2373	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2374	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2375	#endif
2376	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2377
2378	/*
2379	* Host GDTR and IDTR.
2380	*/
2381	RTGDTR Gdtr;
2382	RT_ZERO(Gdtr);
2383	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2384	if (HMVMX_IS_64BIT_HOST_MODE())
2385	{
2386	X86XDTR64 Gdtr64;
2387	X86XDTR64 Idtr64;
2388	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2389	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2390	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2391
2392	Gdtr.cbGdt = Gdtr64.cb;
2393	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2394	}
2395	else
2396	#endif
2397	{
2398	RTIDTR Idtr;
2399	ASMGetGDTR(&Gdtr);
2400	ASMGetIDTR(&Idtr);
2401	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2402	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2403
2404	#if HC_ARCH_BITS == 64
2405	/*
2406	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2407	* maximum limit (0xffff) on every VM-exit.
2408	*/
2409	if (Gdtr.cbGdt != 0xffff)
2410	{
2411	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2412	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2413	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2414	}
2415
2416	/*
2417	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2418	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2419	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2420	*/
2421	if (Idtr.cbIdt < 0x0fff)
2422	{
2423	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2424	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2425	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2426	}
2427	#endif
2428	}
2429
2430	/*
2431	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2432	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2433	*/
2434	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2435	{
2436	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2437	return VERR_VMX_INVALID_HOST_STATE;
2438	}
2439
2440	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2441	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2442	if (HMVMX_IS_64BIT_HOST_MODE())
2443	{
2444	/* We need the 64-bit TR base for hybrid darwin. */
2445	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2446	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2447	}
2448	else
2449	#endif
2450	{
2451	uintptr_t uTRBase;
2452	#if HC_ARCH_BITS == 64
2453	uTRBase = X86DESC64_BASE(pDesc);
2454
2455	/*
2456	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2457	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2458	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2459	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2460	*
2461	* [1] See Intel spec. 3.5 "System Descriptor Types".
2462	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2463	*/
2464	Assert(pDesc->System.u4Type == 11);
2465	if ( pDesc->System.u16LimitLow != 0x67
2466	\|\| pDesc->System.u4LimitHigh)
2467	{
2468	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2469	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2470
2471	/* Store the GDTR here as we need it while restoring TR. */
2472	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2473	}
2474	#else
2475	uTRBase = X86DESC_BASE(pDesc);
2476	#endif
2477	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2478	}
2479	AssertRCReturn(rc, rc);
2480
2481	/*
2482	* Host FS base and GS base.
2483	*/
2484	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2485	if (HMVMX_IS_64BIT_HOST_MODE())
2486	{
2487	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2488	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2489	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2490	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2491
2492	# if HC_ARCH_BITS == 64
2493	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2494	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2495	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2496	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2497	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2498	# endif
2499	}
2500	#endif
2501	return rc;
2502	}
2503
2504
2505	/**
2506	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2507	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2508	* the host after every successful VM exit.
2509	*
2510	* @returns VBox status code.
2511	* @param pVM Pointer to the VM.
2512	* @param pVCpu Pointer to the VMCPU.
2513	*/
2514	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2515	{
2516	AssertPtr(pVCpu);
2517	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2518
2519	int rc = VINF_SUCCESS;
2520	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2521	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2522	uint32_t cHostMsrs = 0;
2523	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2524
2525	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2526	{
2527	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2528
2529	# if HC_ARCH_BITS == 64
2530	/* Paranoia. 64-bit code requires these bits to be set always. */
2531	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2532
2533	/*
2534	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2535	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2536	* some reason (e.g. allow transparent reads) we would activate the code below.
2537	*/
2538	# if 0
2539	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2540	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2541	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2542	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2543	if (CPUMIsGuestInLongMode(pVCpu))
2544	{
2545	uint64_t u64GuestEfer;
2546	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2547	AssertRC(rc);
2548
2549	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2550	{
2551	pHostMsr->u32Msr = MSR_K6_EFER;
2552	pHostMsr->u32Reserved = 0;
2553	pHostMsr->u64Value = u64HostEfer;
2554	pHostMsr++; cHostMsrs++;
2555	}
2556	}
2557	# endif
2558	# else /* HC_ARCH_BITS != 64 */
2559	pHostMsr->u32Msr = MSR_K6_EFER;
2560	pHostMsr->u32Reserved = 0;
2561	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2562	if (CPUMIsGuestInLongMode(pVCpu))
2563	{
2564	/* Must match the EFER value in our 64 bits switcher. */
2565	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2566	}
2567	else
2568	# endif
2569	pHostMsr->u64Value = u64HostEfer;
2570	pHostMsr++; cHostMsrs++;
2571	# endif /* HC_ARCH_BITS == 64 */
2572	}
2573
2574	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2575	if (HMVMX_IS_64BIT_HOST_MODE())
2576	{
2577	pHostMsr->u32Msr = MSR_K6_STAR;
2578	pHostMsr->u32Reserved = 0;
2579	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2580	pHostMsr++; cHostMsrs++;
2581	pHostMsr->u32Msr = MSR_K8_LSTAR;
2582	pHostMsr->u32Reserved = 0;
2583	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2584	pHostMsr++; cHostMsrs++;
2585	pHostMsr->u32Msr = MSR_K8_SF_MASK;
2586	pHostMsr->u32Reserved = 0;
2587	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2588	pHostMsr++; cHostMsrs++;
2589	pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2590	pHostMsr->u32Reserved = 0;
2591	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2592	pHostMsr++; cHostMsrs++;
2593	}
2594	# endif
2595
2596	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2597	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2598	{
2599	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2600	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2601	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2602	}
2603
2604	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2605	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2606
2607	/*
2608	* Host Sysenter MSRs.
2609	*/
2610	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2611	AssertRCReturn(rc, rc);
2612	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2613	if (HMVMX_IS_64BIT_HOST_MODE())
2614	{
2615	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2616	AssertRCReturn(rc, rc);
2617	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2618	}
2619	else
2620	{
2621	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2622	AssertRCReturn(rc, rc);
2623	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2624	}
2625	#elif HC_ARCH_BITS == 32
2626	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2627	AssertRCReturn(rc, rc);
2628	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2629	#else
2630	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2631	AssertRCReturn(rc, rc);
2632	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2633	#endif
2634	AssertRCReturn(rc, rc);
2635
2636	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2637	* hmR0VmxSetupExitCtls() !! */
2638	return rc;
2639	}
2640
2641
2642	/**
2643	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2644	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2645	* controls".
2646	*
2647	* @returns VBox status code.
2648	* @param pVCpu Pointer to the VMCPU.
2649	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2650	* out-of-sync. Make sure to update the required fields
2651	* before using them.
2652	*
2653	* @remarks No-long-jump zone!!!
2654	*/
2655	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2656	{
2657	int rc = VINF_SUCCESS;
2658	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2659	{
2660	PVM pVM = pVCpu->CTX_SUFF(pVM);
2661	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2662	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2663
2664	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2665	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2666
2667	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2668	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2669	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2670	else
2671	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2672
2673	/*
2674	* The following should -not- be set (since we're not in SMM mode):
2675	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2676	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2677	*/
2678
2679	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2680	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2681	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2682
2683	if ((val & zap) != val)
2684	{
2685	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2686	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2687	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2688	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2689	}
2690
2691	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2692	AssertRCReturn(rc, rc);
2693
2694	/* Update VCPU with the currently set VM-exit controls. */
2695	pVCpu->hm.s.vmx.u32EntryCtls = val;
2696	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2697	}
2698	return rc;
2699	}
2700
2701
2702	/**
2703	* Sets up the VM-exit controls in the VMCS.
2704	*
2705	* @returns VBox status code.
2706	* @param pVM Pointer to the VM.
2707	* @param pVCpu Pointer to the VMCPU.
2708	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2709	* out-of-sync. Make sure to update the required fields
2710	* before using them.
2711	*
2712	* @remarks requires EFER.
2713	*/
2714	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2715	{
2716	int rc = VINF_SUCCESS;
2717	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2718	{
2719	PVM pVM = pVCpu->CTX_SUFF(pVM);
2720	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2721	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2722
2723	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2724	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2725
2726	/*
2727	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2728	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2729	*/
2730	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2731	if (HMVMX_IS_64BIT_HOST_MODE())
2732	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2733	else
2734	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2735	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2736	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2737	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2738	else
2739	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2740	#endif
2741
2742	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2743	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2744
2745	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2746	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2747	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2748	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2749	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2750
2751	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2752	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2753
2754	if ((val & zap) != val)
2755	{
2756	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2757	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2758	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2759	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2760	}
2761
2762	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2763	AssertRCReturn(rc, rc);
2764
2765	/* Update VCPU with the currently set VM-exit controls. */
2766	pVCpu->hm.s.vmx.u32ExitCtls = val;
2767	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2768	}
2769	return rc;
2770	}
2771
2772
2773	/**
2774	* Loads the guest APIC and related state.
2775	*
2776	* @returns VBox status code.
2777	* @param pVM Pointer to the VM.
2778	* @param pVCpu Pointer to the VMCPU.
2779	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2780	* out-of-sync. Make sure to update the required fields
2781	* before using them.
2782	*/
2783	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2784	{
2785	int rc = VINF_SUCCESS;
2786	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2787	{
2788	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2789	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2790	{
2791	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2792
2793	bool fPendingIntr = false;
2794	uint8_t u8Tpr = 0;
2795	uint8_t u8PendingIntr = 0;
2796	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2797	AssertRCReturn(rc, rc);
2798
2799	/*
2800	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2801	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2802	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2803	* the interrupt when we VM-exit for other reasons.
2804	*/
2805	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2806	uint32_t u32TprThreshold = 0;
2807	if (fPendingIntr)
2808	{
2809	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2810	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2811	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2812	if (u8PendingPriority <= u8TprPriority)
2813	u32TprThreshold = u8PendingPriority;
2814	else
2815	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2816	}
2817	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2818
2819	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2820	AssertRCReturn(rc, rc);
2821	}
2822
2823	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
2824	}
2825	return rc;
2826	}
2827
2828
2829	/**
2830	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2831	*
2832	* @returns Guest's interruptibility-state.
2833	* @param pVCpu Pointer to the VMCPU.
2834	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2835	* out-of-sync. Make sure to update the required fields
2836	* before using them.
2837	*
2838	* @remarks No-long-jump zone!!!
2839	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2840	*/
2841	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2842	{
2843	/*
2844	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2845	* inhibit interrupts or clear any existing interrupt-inhibition.
2846	*/
2847	uint32_t uIntrState = 0;
2848	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2849	{
2850	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2851	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2852	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2853	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2854	{
2855	/*
2856	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2857	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2858	*/
2859	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2860	}
2861	else if (pMixedCtx->eflags.Bits.u1IF)
2862	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2863	else
2864	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2865	}
2866	return uIntrState;
2867	}
2868
2869
2870	/**
2871	* Loads the guest's interruptibility-state into the guest-state area in the
2872	* VMCS.
2873	*
2874	* @returns VBox status code.
2875	* @param pVCpu Pointer to the VMCPU.
2876	* @param uIntrState The interruptibility-state to set.
2877	*/
2878	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2879	{
2880	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2881	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2882	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2883	AssertRCReturn(rc, rc);
2884	return rc;
2885	}
2886
2887
2888	/**
2889	* Loads the guest's RIP into the guest-state area in the VMCS.
2890	*
2891	* @returns VBox status code.
2892	* @param pVCpu Pointer to the VMCPU.
2893	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2894	* out-of-sync. Make sure to update the required fields
2895	* before using them.
2896	*
2897	* @remarks No-long-jump zone!!!
2898	*/
2899	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2900	{
2901	int rc = VINF_SUCCESS;
2902	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
2903	{
2904	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2905	AssertRCReturn(rc, rc);
2906
2907	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
2908	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
2909	}
2910	return rc;
2911	}
2912
2913
2914	/**
2915	* Loads the guest's RSP into the guest-state area in the VMCS.
2916	*
2917	* @returns VBox status code.
2918	* @param pVCpu Pointer to the VMCPU.
2919	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2920	* out-of-sync. Make sure to update the required fields
2921	* before using them.
2922	*
2923	* @remarks No-long-jump zone!!!
2924	*/
2925	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2926	{
2927	int rc = VINF_SUCCESS;
2928	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
2929	{
2930	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2931	AssertRCReturn(rc, rc);
2932
2933	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
2934	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2935	}
2936	return rc;
2937	}
2938
2939
2940	/**
2941	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2942	*
2943	* @returns VBox status code.
2944	* @param pVCpu Pointer to the VMCPU.
2945	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2946	* out-of-sync. Make sure to update the required fields
2947	* before using them.
2948	*
2949	* @remarks No-long-jump zone!!!
2950	*/
2951	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2952	{
2953	int rc = VINF_SUCCESS;
2954	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
2955	{
2956	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2957	Let us assert it as such and use 32-bit VMWRITE. */
2958	Assert(!(pMixedCtx->rflags.u64 >> 32));
2959	X86EFLAGS Eflags = pMixedCtx->eflags;
2960	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2961	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2962
2963	/*
2964	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2965	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2966	*/
2967	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2968	{
2969	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2970	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2971	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2972	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2973	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2974	}
2975
2976	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2977	AssertRCReturn(rc, rc);
2978
2979	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
2980	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2981	}
2982	return rc;
2983	}
2984
2985
2986	/**
2987	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2988	*
2989	* @returns VBox status code.
2990	* @param pVCpu Pointer to the VMCPU.
2991	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2992	* out-of-sync. Make sure to update the required fields
2993	* before using them.
2994	*
2995	* @remarks No-long-jump zone!!!
2996	*/
2997	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2998	{
2999	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3000	AssertRCReturn(rc, rc);
3001	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3002	AssertRCReturn(rc, rc);
3003	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3004	AssertRCReturn(rc, rc);
3005	return rc;
3006	}
3007
3008
3009	/**
3010	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3011	* CR0 is partially shared with the host and we have to consider the FPU bits.
3012	*
3013	* @returns VBox status code.
3014	* @param pVM Pointer to the VM.
3015	* @param pVCpu Pointer to the VMCPU.
3016	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3017	* out-of-sync. Make sure to update the required fields
3018	* before using them.
3019	*
3020	* @remarks No-long-jump zone!!!
3021	*/
3022	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3023	{
3024	/*
3025	* Guest CR0.
3026	* Guest FPU.
3027	*/
3028	int rc = VINF_SUCCESS;
3029	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3030	{
3031	Assert(!(pMixedCtx->cr0 >> 32));
3032	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3033	PVM pVM = pVCpu->CTX_SUFF(pVM);
3034
3035	/* The guest's view (read access) of its CR0 is unblemished. */
3036	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3037	AssertRCReturn(rc, rc);
3038	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3039
3040	/* Setup VT-x's view of the guest CR0. */
3041	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3042	if (pVM->hm.s.fNestedPaging)
3043	{
3044	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3045	{
3046	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3047	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3048	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3049	}
3050	else
3051	{
3052	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3053	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3054	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3055	}
3056
3057	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3058	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3059	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3060
3061	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3062	AssertRCReturn(rc, rc);
3063	}
3064	else
3065	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3066
3067	/*
3068	* Guest FPU bits.
3069	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3070	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3071	*/
3072	u32GuestCR0 \|= X86_CR0_NE;
3073	bool fInterceptNM = false;
3074	if (CPUMIsGuestFPUStateActive(pVCpu))
3075	{
3076	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3077	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3078	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3079	}
3080	else
3081	{
3082	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3083	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3084	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3085	}
3086
3087	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3088	bool fInterceptMF = false;
3089	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3090	fInterceptMF = true;
3091
3092	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3093	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3094	{
3095	Assert(PDMVmmDevHeapIsEnabled(pVM));
3096	Assert(pVM->hm.s.vmx.pRealModeTSS);
3097	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3098	fInterceptNM = true;
3099	fInterceptMF = true;
3100	}
3101	else
3102	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3103
3104	if (fInterceptNM)
3105	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3106	else
3107	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3108
3109	if (fInterceptMF)
3110	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3111	else
3112	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3113
3114	/* Additional intercepts for debugging, define these yourself explicitly. */
3115	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3116	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3117	\| RT_BIT(X86_XCPT_BP)
3118	\| RT_BIT(X86_XCPT_DB)
3119	\| RT_BIT(X86_XCPT_DE)
3120	\| RT_BIT(X86_XCPT_NM)
3121	\| RT_BIT(X86_XCPT_UD)
3122	\| RT_BIT(X86_XCPT_NP)
3123	\| RT_BIT(X86_XCPT_SS)
3124	\| RT_BIT(X86_XCPT_GP)
3125	\| RT_BIT(X86_XCPT_PF)
3126	\| RT_BIT(X86_XCPT_MF)
3127	;
3128	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3129	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3130	#endif
3131
3132	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3133
3134	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3135	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3136	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3137	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3138	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3139	else
3140	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3141
3142	u32GuestCR0 \|= uSetCR0;
3143	u32GuestCR0 &= uZapCR0;
3144	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3145
3146	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3147	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3148	AssertRCReturn(rc, rc);
3149	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3150	AssertRCReturn(rc, rc);
3151	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3152
3153	/*
3154	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3155	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3156	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3157	*/
3158	uint32_t u32CR0Mask = 0;
3159	u32CR0Mask = X86_CR0_PE
3160	\| X86_CR0_NE
3161	\| X86_CR0_WP
3162	\| X86_CR0_PG
3163	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3164	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3165	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3166
3167	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3168	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3169	* and @bugref{6944}. */
3170	#if 0
3171	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3172	u32CR0Mask &= ~X86_CR0_PE;
3173	#endif
3174	if (pVM->hm.s.fNestedPaging)
3175	u32CR0Mask &= ~X86_CR0_WP;
3176
3177	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3178	if (fInterceptNM)
3179	{
3180	u32CR0Mask \|= X86_CR0_TS
3181	\| X86_CR0_MP;
3182	}
3183
3184	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3185	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3186	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3187	AssertRCReturn(rc, rc);
3188	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3189
3190	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3191	}
3192	return rc;
3193	}
3194
3195
3196	/**
3197	* Loads the guest control registers (CR3, CR4) into the guest-state area
3198	* in the VMCS.
3199	*
3200	* @returns VBox status code.
3201	* @param pVM Pointer to the VM.
3202	* @param pVCpu Pointer to the VMCPU.
3203	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3204	* out-of-sync. Make sure to update the required fields
3205	* before using them.
3206	*
3207	* @remarks No-long-jump zone!!!
3208	*/
3209	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3210	{
3211	int rc = VINF_SUCCESS;
3212	PVM pVM = pVCpu->CTX_SUFF(pVM);
3213
3214	/*
3215	* Guest CR2.
3216	* It's always loaded in the assembler code. Nothing to do here.
3217	*/
3218
3219	/*
3220	* Guest CR3.
3221	*/
3222	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3223	{
3224	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3225	if (pVM->hm.s.fNestedPaging)
3226	{
3227	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3228
3229	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3230	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3231	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3232	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3233
3234	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3235	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3236	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3237
3238	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3239	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3240	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3241	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3242
3243	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3244	AssertRCReturn(rc, rc);
3245	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3246
3247	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3248	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3249	{
3250	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3251	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3252	{
3253	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3254	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3255	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3256	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3257	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3258	}
3259
3260	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3261	have Unrestricted Execution to handle the guest when it's not using paging. */
3262	GCPhysGuestCR3 = pMixedCtx->cr3;
3263	}
3264	else
3265	{
3266	/*
3267	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3268	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3269	* EPT takes care of translating it to host-physical addresses.
3270	*/
3271	RTGCPHYS GCPhys;
3272	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3273	Assert(PDMVmmDevHeapIsEnabled(pVM));
3274
3275	/* We obtain it here every time as the guest could have relocated this PCI region. */
3276	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3277	AssertRCReturn(rc, rc);
3278
3279	GCPhysGuestCR3 = GCPhys;
3280	}
3281
3282	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3283	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3284	}
3285	else
3286	{
3287	/* Non-nested paging case, just use the hypervisor's CR3. */
3288	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3289
3290	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3291	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3292	}
3293	AssertRCReturn(rc, rc);
3294
3295	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3296	}
3297
3298	/*
3299	* Guest CR4.
3300	*/
3301	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3302	{
3303	Assert(!(pMixedCtx->cr4 >> 32));
3304	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3305
3306	/* The guest's view of its CR4 is unblemished. */
3307	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3308	AssertRCReturn(rc, rc);
3309	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3310
3311	/* Setup VT-x's view of the guest CR4. */
3312	/*
3313	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3314	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3315	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3316	*/
3317	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3318	{
3319	Assert(pVM->hm.s.vmx.pRealModeTSS);
3320	Assert(PDMVmmDevHeapIsEnabled(pVM));
3321	u32GuestCR4 &= ~X86_CR4_VME;
3322	}
3323
3324	if (pVM->hm.s.fNestedPaging)
3325	{
3326	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3327	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3328	{
3329	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3330	u32GuestCR4 \|= X86_CR4_PSE;
3331	/* Our identity mapping is a 32 bits page directory. */
3332	u32GuestCR4 &= ~X86_CR4_PAE;
3333	}
3334	/* else use guest CR4.*/
3335	}
3336	else
3337	{
3338	/*
3339	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3340	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3341	*/
3342	switch (pVCpu->hm.s.enmShadowMode)
3343	{
3344	case PGMMODE_REAL: /* Real-mode. */
3345	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3346	case PGMMODE_32_BIT: /* 32-bit paging. */
3347	{
3348	u32GuestCR4 &= ~X86_CR4_PAE;
3349	break;
3350	}
3351
3352	case PGMMODE_PAE: /* PAE paging. */
3353	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3354	{
3355	u32GuestCR4 \|= X86_CR4_PAE;
3356	break;
3357	}
3358
3359	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3360	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3361	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3362	break;
3363	#endif
3364	default:
3365	AssertFailed();
3366	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3367	}
3368	}
3369
3370	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3371	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3372	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3373	u32GuestCR4 \|= uSetCR4;
3374	u32GuestCR4 &= uZapCR4;
3375
3376	/* Write VT-x's view of the guest CR4 into the VMCS. */
3377	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3378	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3379	AssertRCReturn(rc, rc);
3380
3381	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3382	uint32_t u32CR4Mask = 0;
3383	u32CR4Mask = X86_CR4_VME
3384	\| X86_CR4_PAE
3385	\| X86_CR4_PGE
3386	\| X86_CR4_PSE
3387	\| X86_CR4_VMXE;
3388	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3389	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3390	AssertRCReturn(rc, rc);
3391
3392	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3393	}
3394	return rc;
3395	}
3396
3397
3398	/**
3399	* Loads the guest debug registers into the guest-state area in the VMCS.
3400	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3401	*
3402	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3403	*
3404	* @returns VBox status code.
3405	* @param pVCpu Pointer to the VMCPU.
3406	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3407	* out-of-sync. Make sure to update the required fields
3408	* before using them.
3409	*
3410	* @remarks No-long-jump zone!!!
3411	*/
3412	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3413	{
3414	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3415	return VINF_SUCCESS;
3416
3417	#ifdef VBOX_STRICT
3418	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3419	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3420	{
3421	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3422	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3423	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3424	}
3425	#endif
3426
3427	int rc;
3428	PVM pVM = pVCpu->CTX_SUFF(pVM);
3429	bool fInterceptDB = false;
3430	bool fInterceptMovDRx = false;
3431	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3432	{
3433	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3434	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3435	{
3436	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3437	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3438	AssertRCReturn(rc, rc);
3439	Assert(fInterceptDB == false);
3440	}
3441	else
3442	{
3443	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3444	pVCpu->hm.s.fClearTrapFlag = true;
3445	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3446	fInterceptDB = true;
3447	}
3448	}
3449
3450	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3451	{
3452	/*
3453	* Use the combined guest and host DRx values found in the hypervisor
3454	* register set because the debugger has breakpoints active or someone
3455	* is single stepping on the host side without a monitor trap flag.
3456	*
3457	* Note! DBGF expects a clean DR6 state before executing guest code.
3458	*/
3459	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3460	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3461	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3462	{
3463	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3464	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3465	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3466	}
3467	else
3468	#endif
3469	if (!CPUMIsHyperDebugStateActive(pVCpu))
3470	{
3471	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3472	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3473	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3474	}
3475
3476	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3477	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3478	AssertRCReturn(rc, rc);
3479
3480	pVCpu->hm.s.fUsingHyperDR7 = true;
3481	fInterceptDB = true;
3482	fInterceptMovDRx = true;
3483	}
3484	else
3485	{
3486	/*
3487	* If the guest has enabled debug registers, we need to load them prior to
3488	* executing guest code so they'll trigger at the right time.
3489	*/
3490	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3491	{
3492	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3493	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3494	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3495	{
3496	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3497	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3498	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3499	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3500	}
3501	else
3502	#endif
3503	if (CPUMIsGuestDebugStateActive(pVCpu))
3504	{
3505	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3506	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3507	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3508	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3509	}
3510	}
3511	/*
3512	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3513	* must intercept #DB in order to maintain a correct DR6 guest value.
3514	*/
3515	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3516	else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3517	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3518	\|\| !CPUMIsGuestDebugStateActive(pVCpu))
3519	#else
3520	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3521	#endif
3522	{
3523	fInterceptMovDRx = true;
3524	fInterceptDB = true;
3525	}
3526
3527	/* Update guest DR7. */
3528	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3529	AssertRCReturn(rc, rc);
3530
3531	pVCpu->hm.s.fUsingHyperDR7 = false;
3532	}
3533
3534	/*
3535	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3536	*/
3537	if (fInterceptDB)
3538	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3539	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3540	{
3541	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3542	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3543	#endif
3544	}
3545	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3546	AssertRCReturn(rc, rc);
3547
3548	/*
3549	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3550	*/
3551	if (fInterceptMovDRx)
3552	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3553	else
3554	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3555	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3556	AssertRCReturn(rc, rc);
3557
3558	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3559	return VINF_SUCCESS;
3560	}
3561
3562
3563	#ifdef VBOX_STRICT
3564	/**
3565	* Strict function to validate segment registers.
3566	*
3567	* @remarks ASSUMES CR0 is up to date.
3568	*/
3569	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3570	{
3571	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3572	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3573	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3574	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3575	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3576	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3577	{
3578	/* Protected mode checks */
3579	/* CS */
3580	Assert(pCtx->cs.Attr.n.u1Present);
3581	Assert(!(pCtx->cs.Attr.u & 0xf00));
3582	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3583	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3584	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3585	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3586	\|\| (pCtx->cs.Attr.n.u1Granularity));
3587	/* CS cannot be loaded with NULL in protected mode. */
3588	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3589	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3590	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3591	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3592	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3593	else
3594	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3595	/* SS */
3596	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3597	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3598	if ( !(pCtx->cr0 & X86_CR0_PE)
3599	\|\| pCtx->cs.Attr.n.u4Type == 3)
3600	{
3601	Assert(!pCtx->ss.Attr.n.u2Dpl);
3602	}
3603	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3604	{
3605	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3606	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3607	Assert(pCtx->ss.Attr.n.u1Present);
3608	Assert(!(pCtx->ss.Attr.u & 0xf00));
3609	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3610	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3611	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3612	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3613	\|\| (pCtx->ss.Attr.n.u1Granularity));
3614	}
3615	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3616	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3617	{
3618	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3619	Assert(pCtx->ds.Attr.n.u1Present);
3620	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3621	Assert(!(pCtx->ds.Attr.u & 0xf00));
3622	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3623	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3624	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3625	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3626	\|\| (pCtx->ds.Attr.n.u1Granularity));
3627	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3628	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3629	}
3630	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3631	{
3632	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3633	Assert(pCtx->es.Attr.n.u1Present);
3634	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3635	Assert(!(pCtx->es.Attr.u & 0xf00));
3636	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3637	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3638	\|\| !(pCtx->es.Attr.n.u1Granularity));
3639	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3640	\|\| (pCtx->es.Attr.n.u1Granularity));
3641	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3642	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3643	}
3644	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3645	{
3646	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3647	Assert(pCtx->fs.Attr.n.u1Present);
3648	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3649	Assert(!(pCtx->fs.Attr.u & 0xf00));
3650	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3651	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3652	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3653	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3654	\|\| (pCtx->fs.Attr.n.u1Granularity));
3655	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3656	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3657	}
3658	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3659	{
3660	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3661	Assert(pCtx->gs.Attr.n.u1Present);
3662	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3663	Assert(!(pCtx->gs.Attr.u & 0xf00));
3664	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3665	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3666	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3667	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3668	\|\| (pCtx->gs.Attr.n.u1Granularity));
3669	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3670	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3671	}
3672	/* 64-bit capable CPUs. */
3673	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3674	Assert(!(pCtx->cs.u64Base >> 32));
3675	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3676	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3677	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3678	# endif
3679	}
3680	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3681	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3682	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3683	{
3684	/* Real and v86 mode checks. */
3685	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3686	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3687	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3688	{
3689	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3690	}
3691	else
3692	{
3693	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3694	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3695	}
3696
3697	/* CS */
3698	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3699	Assert(pCtx->cs.u32Limit == 0xffff);
3700	Assert(u32CSAttr == 0xf3);
3701	/* SS */
3702	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3703	Assert(pCtx->ss.u32Limit == 0xffff);
3704	Assert(u32SSAttr == 0xf3);
3705	/* DS */
3706	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3707	Assert(pCtx->ds.u32Limit == 0xffff);
3708	Assert(u32DSAttr == 0xf3);
3709	/* ES */
3710	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3711	Assert(pCtx->es.u32Limit == 0xffff);
3712	Assert(u32ESAttr == 0xf3);
3713	/* FS */
3714	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3715	Assert(pCtx->fs.u32Limit == 0xffff);
3716	Assert(u32FSAttr == 0xf3);
3717	/* GS */
3718	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3719	Assert(pCtx->gs.u32Limit == 0xffff);
3720	Assert(u32GSAttr == 0xf3);
3721	/* 64-bit capable CPUs. */
3722	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3723	Assert(!(pCtx->cs.u64Base >> 32));
3724	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3725	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3726	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3727	# endif
3728	}
3729	}
3730	#endif /* VBOX_STRICT */
3731
3732
3733	/**
3734	* Writes a guest segment register into the guest-state area in the VMCS.
3735	*
3736	* @returns VBox status code.
3737	* @param pVCpu Pointer to the VMCPU.
3738	* @param idxSel Index of the selector in the VMCS.
3739	* @param idxLimit Index of the segment limit in the VMCS.
3740	* @param idxBase Index of the segment base in the VMCS.
3741	* @param idxAccess Index of the access rights of the segment in the VMCS.
3742	* @param pSelReg Pointer to the segment selector.
3743	* @param pCtx Pointer to the guest-CPU context.
3744	*
3745	* @remarks No-long-jump zone!!!
3746	*/
3747	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3748	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3749	{
3750	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3751	AssertRCReturn(rc, rc);
3752	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3753	AssertRCReturn(rc, rc);
3754	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3755	AssertRCReturn(rc, rc);
3756
3757	uint32_t u32Access = pSelReg->Attr.u;
3758	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3759	{
3760	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3761	u32Access = 0xf3;
3762	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3763	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3764	}
3765	else
3766	{
3767	/*
3768	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3769	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3770	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3771	* loaded in protected-mode have their attribute as 0.
3772	*/
3773	if (!u32Access)
3774	u32Access = X86DESCATTR_UNUSABLE;
3775	}
3776
3777	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3778	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3779	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3780
3781	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3782	AssertRCReturn(rc, rc);
3783	return rc;
3784	}
3785
3786
3787	/**
3788	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3789	* into the guest-state area in the VMCS.
3790	*
3791	* @returns VBox status code.
3792	* @param pVM Pointer to the VM.
3793	* @param pVCPU Pointer to the VMCPU.
3794	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3795	* out-of-sync. Make sure to update the required fields
3796	* before using them.
3797	*
3798	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3799	* @remarks No-long-jump zone!!!
3800	*/
3801	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3802	{
3803	int rc = VERR_INTERNAL_ERROR_5;
3804	PVM pVM = pVCpu->CTX_SUFF(pVM);
3805
3806	/*
3807	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3808	*/
3809	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3810	{
3811	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3812	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3813	{
3814	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3815	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3816	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3817	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3818	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3819	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3820	}
3821
3822	#ifdef VBOX_WITH_REM
3823	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3824	{
3825	Assert(pVM->hm.s.vmx.pRealModeTSS);
3826	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3827	if ( pVCpu->hm.s.vmx.fWasInRealMode
3828	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3829	{
3830	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3831	in real-mode (e.g. OpenBSD 4.0) */
3832	REMFlushTBs(pVM);
3833	Log4(("Load: Switch to protected mode detected!\n"));
3834	pVCpu->hm.s.vmx.fWasInRealMode = false;
3835	}
3836	}
3837	#endif
3838	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3839	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3840	AssertRCReturn(rc, rc);
3841	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3842	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3843	AssertRCReturn(rc, rc);
3844	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3845	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3846	AssertRCReturn(rc, rc);
3847	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3848	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3849	AssertRCReturn(rc, rc);
3850	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3851	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3852	AssertRCReturn(rc, rc);
3853	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3854	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3855	AssertRCReturn(rc, rc);
3856
3857	#ifdef VBOX_STRICT
3858	/* Validate. */
3859	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3860	#endif
3861
3862	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
3863	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3864	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3865	}
3866
3867	/*
3868	* Guest TR.
3869	*/
3870	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
3871	{
3872	/*
3873	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3874	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3875	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3876	*/
3877	uint16_t u16Sel = 0;
3878	uint32_t u32Limit = 0;
3879	uint64_t u64Base = 0;
3880	uint32_t u32AccessRights = 0;
3881
3882	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3883	{
3884	u16Sel = pMixedCtx->tr.Sel;
3885	u32Limit = pMixedCtx->tr.u32Limit;
3886	u64Base = pMixedCtx->tr.u64Base;
3887	u32AccessRights = pMixedCtx->tr.Attr.u;
3888	}
3889	else
3890	{
3891	Assert(pVM->hm.s.vmx.pRealModeTSS);
3892	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3893
3894	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3895	RTGCPHYS GCPhys;
3896	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3897	AssertRCReturn(rc, rc);
3898
3899	X86DESCATTR DescAttr;
3900	DescAttr.u = 0;
3901	DescAttr.n.u1Present = 1;
3902	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3903
3904	u16Sel = 0;
3905	u32Limit = HM_VTX_TSS_SIZE;
3906	u64Base = GCPhys; /* in real-mode phys = virt. */
3907	u32AccessRights = DescAttr.u;
3908	}
3909
3910	/* Validate. */
3911	Assert(!(u16Sel & RT_BIT(2)));
3912	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3913	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3914	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3915	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3916	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3917	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3918	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3919	Assert( (u32Limit & 0xfff) == 0xfff
3920	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3921	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3922	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3923
3924	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3925	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3926	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3927	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3928
3929	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
3930	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3931	}
3932
3933	/*
3934	* Guest GDTR.
3935	*/
3936	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
3937	{
3938	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3939	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3940
3941	/* Validate. */
3942	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3943
3944	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
3945	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3946	}
3947
3948	/*
3949	* Guest LDTR.
3950	*/
3951	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
3952	{
3953	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3954	uint32_t u32Access = 0;
3955	if (!pMixedCtx->ldtr.Attr.u)
3956	u32Access = X86DESCATTR_UNUSABLE;
3957	else
3958	u32Access = pMixedCtx->ldtr.Attr.u;
3959
3960	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3961	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3962	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3963	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3964
3965	/* Validate. */
3966	if (!(u32Access & X86DESCATTR_UNUSABLE))
3967	{
3968	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3969	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3970	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3971	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3972	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3973	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3974	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3975	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3976	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3977	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3978	}
3979
3980	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
3981	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3982	}
3983
3984	/*
3985	* Guest IDTR.
3986	*/
3987	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
3988	{
3989	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3990	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3991
3992	/* Validate. */
3993	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3994
3995	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
3996	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3997	}
3998
3999	return VINF_SUCCESS;
4000	}
4001
4002
4003	/**
4004	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4005	* areas. These MSRs will automatically be loaded to the host CPU on every
4006	* successful VM entry and stored from the host CPU on every successful VM exit.
4007	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4008	*
4009	* @returns VBox status code.
4010	* @param pVCpu Pointer to the VMCPU.
4011	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4012	* out-of-sync. Make sure to update the required fields
4013	* before using them.
4014	*
4015	* @remarks No-long-jump zone!!!
4016	*/
4017	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4018	{
4019	AssertPtr(pVCpu);
4020	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4021
4022	/*
4023	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4024	*/
4025	int rc = VINF_SUCCESS;
4026	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4027	{
4028	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4029	PVM pVM = pVCpu->CTX_SUFF(pVM);
4030	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4031	uint32_t cGuestMsrs = 0;
4032
4033	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4034	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4035	* when the guest really is in 64-bit mode. */
4036	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4037	if (fSupportsLongMode)
4038	{
4039	pGuestMsr->u32Msr = MSR_K8_LSTAR;
4040	pGuestMsr->u32Reserved = 0;
4041	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4042	pGuestMsr++; cGuestMsrs++;
4043	pGuestMsr->u32Msr = MSR_K6_STAR;
4044	pGuestMsr->u32Reserved = 0;
4045	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4046	pGuestMsr++; cGuestMsrs++;
4047	pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4048	pGuestMsr->u32Reserved = 0;
4049	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4050	pGuestMsr++; cGuestMsrs++;
4051	pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4052	pGuestMsr->u32Reserved = 0;
4053	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4054	pGuestMsr++; cGuestMsrs++;
4055	}
4056
4057	/*
4058	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4059	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
4060	*/
4061	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
4062	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
4063	{
4064	pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4065	pGuestMsr->u32Reserved = 0;
4066	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4067	AssertRCReturn(rc, rc);
4068	pGuestMsr++; cGuestMsrs++;
4069	}
4070
4071	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4072	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4073	{
4074	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4075	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4076	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4077	}
4078
4079	/* Update the VCPU's copy of the guest MSR count. */
4080	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4081	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4082	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4083	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4084
4085	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4086	}
4087
4088	/*
4089	* Guest Sysenter MSRs.
4090	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4091	* VM-exits on WRMSRs for these MSRs.
4092	*/
4093	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4094	{
4095	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4096	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4097	}
4098
4099	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4100	{
4101	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4102	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4103	}
4104
4105	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4106	{
4107	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4108	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4109	}
4110
4111	return rc;
4112	}
4113
4114
4115	/**
4116	* Loads the guest activity state into the guest-state area in the VMCS.
4117	*
4118	* @returns VBox status code.
4119	* @param pVCpu Pointer to the VMCPU.
4120	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4121	* out-of-sync. Make sure to update the required fields
4122	* before using them.
4123	*
4124	* @remarks No-long-jump zone!!!
4125	*/
4126	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4127	{
4128	/** @todo See if we can make use of other states, e.g.
4129	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4130	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4131	{
4132	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4133	AssertRCReturn(rc, rc);
4134
4135	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4136	}
4137	return VINF_SUCCESS;
4138	}
4139
4140
4141	/**
4142	* Sets up the appropriate function to run guest code.
4143	*
4144	* @returns VBox status code.
4145	* @param pVCpu Pointer to the VMCPU.
4146	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4147	* out-of-sync. Make sure to update the required fields
4148	* before using them.
4149	*
4150	* @remarks No-long-jump zone!!!
4151	*/
4152	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4153	{
4154	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4155	{
4156	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4157	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4158	#endif
4159	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4160	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4161	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4162	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4163	{
4164	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4165	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4166	}
4167	#else
4168	/* 64-bit host or hybrid host. */
4169	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4170	#endif
4171	}
4172	else
4173	{
4174	/* Guest is not in long mode, use the 32-bit handler. */
4175	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4176	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4177	{
4178	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4179	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4180	}
4181	#else
4182	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4183	#endif
4184	}
4185	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4186	return VINF_SUCCESS;
4187	}
4188
4189
4190	/**
4191	* Wrapper for running the guest code in VT-x.
4192	*
4193	* @returns VBox strict status code.
4194	* @param pVM Pointer to the VM.
4195	* @param pVCpu Pointer to the VMCPU.
4196	* @param pCtx Pointer to the guest-CPU context.
4197	*
4198	* @remarks No-long-jump zone!!!
4199	*/
4200	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4201	{
4202	/*
4203	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4204	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4205	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4206	*/
4207	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4208	/** @todo Add stats for resume vs launch. */
4209	#ifdef VBOX_WITH_KERNEL_USING_XMM
4210	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4211	#else
4212	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4213	#endif
4214	}
4215
4216
4217	/**
4218	* Reports world-switch error and dumps some useful debug info.
4219	*
4220	* @param pVM Pointer to the VM.
4221	* @param pVCpu Pointer to the VMCPU.
4222	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4223	* @param pCtx Pointer to the guest-CPU context.
4224	* @param pVmxTransient Pointer to the VMX transient structure (only
4225	* exitReason updated).
4226	*/
4227	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4228	{
4229	Assert(pVM);
4230	Assert(pVCpu);
4231	Assert(pCtx);
4232	Assert(pVmxTransient);
4233	HMVMX_ASSERT_PREEMPT_SAFE();
4234
4235	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4236	switch (rcVMRun)
4237	{
4238	case VERR_VMX_INVALID_VMXON_PTR:
4239	AssertFailed();
4240	break;
4241	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4242	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4243	{
4244	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4245	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4246	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4247	AssertRC(rc);
4248
4249	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4250	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4251	Cannot do it here as we may have been long preempted. */
4252
4253	#ifdef VBOX_STRICT
4254	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4255	pVmxTransient->uExitReason));
4256	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4257	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4258	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4259	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4260	else
4261	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4262	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4263	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4264
4265	/* VMX control bits. */
4266	uint32_t u32Val;
4267	uint64_t u64Val;
4268	HMVMXHCUINTREG uHCReg;
4269	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4270	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4271	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4272	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4273	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4274	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4275	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4276	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4277	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4278	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4279	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4280	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4281	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4282	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4283	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4284	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4285	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4286	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4287	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4288	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4289	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4290	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4291	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4292	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4293	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4294	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4295	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4296	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4297	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4298	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4299	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4300	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4301	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4302	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4303	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4304	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4305	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4306	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4307	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4308	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4309	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4310	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4311
4312	/* Guest bits. */
4313	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4314	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4315	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4316	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4317	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4318	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4319	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4320	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4321
4322	/* Host bits. */
4323	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4324	Log4(("Host CR0 %#RHr\n", uHCReg));
4325	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4326	Log4(("Host CR3 %#RHr\n", uHCReg));
4327	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4328	Log4(("Host CR4 %#RHr\n", uHCReg));
4329
4330	RTGDTR HostGdtr;
4331	PCX86DESCHC pDesc;
4332	ASMGetGDTR(&HostGdtr);
4333	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4334	Log4(("Host CS %#08x\n", u32Val));
4335	if (u32Val < HostGdtr.cbGdt)
4336	{
4337	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4338	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4339	}
4340
4341	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4342	Log4(("Host DS %#08x\n", u32Val));
4343	if (u32Val < HostGdtr.cbGdt)
4344	{
4345	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4346	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4347	}
4348
4349	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4350	Log4(("Host ES %#08x\n", u32Val));
4351	if (u32Val < HostGdtr.cbGdt)
4352	{
4353	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4354	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4355	}
4356
4357	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4358	Log4(("Host FS %#08x\n", u32Val));
4359	if (u32Val < HostGdtr.cbGdt)
4360	{
4361	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4362	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4363	}
4364
4365	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4366	Log4(("Host GS %#08x\n", u32Val));
4367	if (u32Val < HostGdtr.cbGdt)
4368	{
4369	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4370	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4371	}
4372
4373	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4374	Log4(("Host SS %#08x\n", u32Val));
4375	if (u32Val < HostGdtr.cbGdt)
4376	{
4377	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4378	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4379	}
4380
4381	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4382	Log4(("Host TR %#08x\n", u32Val));
4383	if (u32Val < HostGdtr.cbGdt)
4384	{
4385	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4386	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4387	}
4388
4389	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4390	Log4(("Host TR Base %#RHv\n", uHCReg));
4391	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4392	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4393	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4394	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4395	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4396	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4397	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4398	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4399	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4400	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4401	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4402	Log4(("Host RSP %#RHv\n", uHCReg));
4403	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4404	Log4(("Host RIP %#RHv\n", uHCReg));
4405	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4406	if (HMVMX_IS_64BIT_HOST_MODE())
4407	{
4408	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4409	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4410	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4411	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4412	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4413	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4414	}
4415	# endif
4416	#endif /* VBOX_STRICT */
4417	break;
4418	}
4419
4420	default:
4421	/* Impossible */
4422	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4423	break;
4424	}
4425	NOREF(pVM);
4426	}
4427
4428
4429	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4430	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4431	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4432	#endif
4433	#ifdef VBOX_STRICT
4434	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4435	{
4436	switch (idxField)
4437	{
4438	case VMX_VMCS_GUEST_RIP:
4439	case VMX_VMCS_GUEST_RSP:
4440	case VMX_VMCS_GUEST_SYSENTER_EIP:
4441	case VMX_VMCS_GUEST_SYSENTER_ESP:
4442	case VMX_VMCS_GUEST_GDTR_BASE:
4443	case VMX_VMCS_GUEST_IDTR_BASE:
4444	case VMX_VMCS_GUEST_CS_BASE:
4445	case VMX_VMCS_GUEST_DS_BASE:
4446	case VMX_VMCS_GUEST_ES_BASE:
4447	case VMX_VMCS_GUEST_FS_BASE:
4448	case VMX_VMCS_GUEST_GS_BASE:
4449	case VMX_VMCS_GUEST_SS_BASE:
4450	case VMX_VMCS_GUEST_LDTR_BASE:
4451	case VMX_VMCS_GUEST_TR_BASE:
4452	case VMX_VMCS_GUEST_CR3:
4453	return true;
4454	}
4455	return false;
4456	}
4457
4458	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4459	{
4460	switch (idxField)
4461	{
4462	/* Read-only fields. */
4463	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4464	return true;
4465	}
4466	/* Remaining readable fields should also be writable. */
4467	return hmR0VmxIsValidWriteField(idxField);
4468	}
4469	#endif /* VBOX_STRICT */
4470
4471
4472	/**
4473	* Executes the specified handler in 64-bit mode.
4474	*
4475	* @returns VBox status code.
4476	* @param pVM Pointer to the VM.
4477	* @param pVCpu Pointer to the VMCPU.
4478	* @param pCtx Pointer to the guest CPU context.
4479	* @param enmOp The operation to perform.
4480	* @param cbParam Number of parameters.
4481	* @param paParam Array of 32-bit parameters.
4482	*/
4483	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4484	uint32_t *paParam)
4485	{
4486	int rc, rc2;
4487	PHMGLOBALCPUINFO pCpu;
4488	RTHCPHYS HCPhysCpuPage;
4489	RTCCUINTREG uOldEflags;
4490
4491	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4492	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4493	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4494	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4495
4496	#ifdef VBOX_STRICT
4497	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4498	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4499
4500	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4501	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4502	#endif
4503
4504	/* Disable interrupts. */
4505	uOldEflags = ASMIntDisableFlags();
4506
4507	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4508	RTCPUID idHostCpu = RTMpCpuId();
4509	CPUMR0SetLApic(pVCpu, idHostCpu);
4510	#endif
4511
4512	pCpu = HMR0GetCurrentCpu();
4513	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4514
4515	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4516	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4517
4518	/* Leave VMX Root Mode. */
4519	VMXDisable();
4520
4521	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4522
4523	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4524	CPUMSetHyperEIP(pVCpu, enmOp);
4525	for (int i = (int)cbParam - 1; i >= 0; i--)
4526	CPUMPushHyper(pVCpu, paParam[i]);
4527
4528	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4529
4530	/* Call the switcher. */
4531	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4532	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4533
4534	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4535	/* Make sure the VMX instructions don't cause #UD faults. */
4536	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4537
4538	/* Re-enter VMX Root Mode */
4539	rc2 = VMXEnable(HCPhysCpuPage);
4540	if (RT_FAILURE(rc2))
4541	{
4542	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4543	ASMSetFlags(uOldEflags);
4544	return rc2;
4545	}
4546
4547	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4548	AssertRC(rc2);
4549	Assert(!(ASMGetFlags() & X86_EFL_IF));
4550	ASMSetFlags(uOldEflags);
4551	return rc;
4552	}
4553
4554
4555	/**
4556	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4557	* supporting 64-bit guests.
4558	*
4559	* @returns VBox status code.
4560	* @param fResume Whether to VMLAUNCH or VMRESUME.
4561	* @param pCtx Pointer to the guest-CPU context.
4562	* @param pCache Pointer to the VMCS cache.
4563	* @param pVM Pointer to the VM.
4564	* @param pVCpu Pointer to the VMCPU.
4565	*/
4566	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4567	{
4568	uint32_t aParam[6];
4569	PHMGLOBALCPUINFO pCpu = NULL;
4570	RTHCPHYS HCPhysCpuPage = 0;
4571	int rc = VERR_INTERNAL_ERROR_5;
4572
4573	pCpu = HMR0GetCurrentCpu();
4574	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4575
4576	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4577	pCache->uPos = 1;
4578	pCache->interPD = PGMGetInterPaeCR3(pVM);
4579	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4580	#endif
4581
4582	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4583	pCache->TestIn.HCPhysCpuPage = 0;
4584	pCache->TestIn.HCPhysVmcs = 0;
4585	pCache->TestIn.pCache = 0;
4586	pCache->TestOut.HCPhysVmcs = 0;
4587	pCache->TestOut.pCache = 0;
4588	pCache->TestOut.pCtx = 0;
4589	pCache->TestOut.eflags = 0;
4590	#endif
4591
4592	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4593	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4594	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4595	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4596	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4597	aParam[5] = 0;
4598
4599	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4600	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4601	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4602	#endif
4603	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4604
4605	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4606	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4607	Assert(pCtx->dr[4] == 10);
4608	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4609	#endif
4610
4611	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4612	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4613	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4614	pVCpu->hm.s.vmx.HCPhysVmcs));
4615	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4616	pCache->TestOut.HCPhysVmcs));
4617	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4618	pCache->TestOut.pCache));
4619	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4620	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4621	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4622	pCache->TestOut.pCtx));
4623	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4624	#endif
4625	return rc;
4626	}
4627
4628
4629	/**
4630	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4631	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4632	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4633	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4634	*
4635	* @returns VBox status code.
4636	* @param pVM Pointer to the VM.
4637	* @param pVCpu Pointer to the VMCPU.
4638	*/
4639	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4640	{
4641	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4642	{ \
4643	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4644	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4645	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4646	++cReadFields; \
4647	}
4648
4649	AssertPtr(pVM);
4650	AssertPtr(pVCpu);
4651	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4652	uint32_t cReadFields = 0;
4653
4654	/*
4655	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4656	* and serve to indicate exceptions to the rules.
4657	*/
4658
4659	/* Guest-natural selector base fields. */
4660	#if 0
4661	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4662	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4663	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4664	#endif
4665	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4666	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4667	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4668	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4669	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4670	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4671	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4672	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4673	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4674	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4675	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4676	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4677	#if 0
4678	/* Unused natural width guest-state fields. */
4679	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4680	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4681	#endif
4682	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4683	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4684
4685	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4686	#if 0
4687	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4688	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4689	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4690	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4691	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4692	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4693	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4694	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4695	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4696	#endif
4697
4698	/* Natural width guest-state fields. */
4699	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4700	#if 0
4701	/* Currently unused field. */
4702	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4703	#endif
4704
4705	if (pVM->hm.s.fNestedPaging)
4706	{
4707	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4708	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4709	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4710	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4711	}
4712	else
4713	{
4714	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4715	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4716	}
4717
4718	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4719	return VINF_SUCCESS;
4720	}
4721
4722
4723	/**
4724	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4725	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4726	* darwin, running 64-bit guests).
4727	*
4728	* @returns VBox status code.
4729	* @param pVCpu Pointer to the VMCPU.
4730	* @param idxField The VMCS field encoding.
4731	* @param u64Val 16, 32 or 64 bits value.
4732	*/
4733	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4734	{
4735	int rc;
4736	switch (idxField)
4737	{
4738	/*
4739	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4740	*/
4741	/* 64-bit Control fields. */
4742	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4743	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4744	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4745	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4746	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4747	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4748	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4749	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4750	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4751	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4752	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4753	case VMX_VMCS64_CTRL_EPTP_FULL:
4754	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4755	/* 64-bit Guest-state fields. */
4756	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4757	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4758	case VMX_VMCS64_GUEST_PAT_FULL:
4759	case VMX_VMCS64_GUEST_EFER_FULL:
4760	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4761	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4762	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4763	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4764	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4765	/* 64-bit Host-state fields. */
4766	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4767	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4768	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4769	{
4770	rc = VMXWriteVmcs32(idxField, u64Val);
4771	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4772	break;
4773	}
4774
4775	/*
4776	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4777	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4778	*/
4779	/* Natural-width Guest-state fields. */
4780	case VMX_VMCS_GUEST_CR3:
4781	case VMX_VMCS_GUEST_ES_BASE:
4782	case VMX_VMCS_GUEST_CS_BASE:
4783	case VMX_VMCS_GUEST_SS_BASE:
4784	case VMX_VMCS_GUEST_DS_BASE:
4785	case VMX_VMCS_GUEST_FS_BASE:
4786	case VMX_VMCS_GUEST_GS_BASE:
4787	case VMX_VMCS_GUEST_LDTR_BASE:
4788	case VMX_VMCS_GUEST_TR_BASE:
4789	case VMX_VMCS_GUEST_GDTR_BASE:
4790	case VMX_VMCS_GUEST_IDTR_BASE:
4791	case VMX_VMCS_GUEST_RSP:
4792	case VMX_VMCS_GUEST_RIP:
4793	case VMX_VMCS_GUEST_SYSENTER_ESP:
4794	case VMX_VMCS_GUEST_SYSENTER_EIP:
4795	{
4796	if (!(u64Val >> 32))
4797	{
4798	/* If this field is 64-bit, VT-x will zero out the top bits. */
4799	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4800	}
4801	else
4802	{
4803	/* Assert that only the 32->64 switcher case should ever come here. */
4804	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4805	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4806	}
4807	break;
4808	}
4809
4810	default:
4811	{
4812	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4813	rc = VERR_INVALID_PARAMETER;
4814	break;
4815	}
4816	}
4817	AssertRCReturn(rc, rc);
4818	return rc;
4819	}
4820
4821
4822	/**
4823	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4824	* hosts (except darwin) for 64-bit guests.
4825	*
4826	* @param pVCpu Pointer to the VMCPU.
4827	* @param idxField The VMCS field encoding.
4828	* @param u64Val 16, 32 or 64 bits value.
4829	*/
4830	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4831	{
4832	AssertPtr(pVCpu);
4833	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4834
4835	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4836	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4837
4838	/* Make sure there are no duplicates. */
4839	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4840	{
4841	if (pCache->Write.aField[i] == idxField)
4842	{
4843	pCache->Write.aFieldVal[i] = u64Val;
4844	return VINF_SUCCESS;
4845	}
4846	}
4847
4848	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4849	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4850	pCache->Write.cValidEntries++;
4851	return VINF_SUCCESS;
4852	}
4853
4854	/* Enable later when the assembly code uses these as callbacks. */
4855	#if 0
4856	/*
4857	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4858	*
4859	* @param pVCpu Pointer to the VMCPU.
4860	* @param pCache Pointer to the VMCS cache.
4861	*
4862	* @remarks No-long-jump zone!!!
4863	*/
4864	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4865	{
4866	AssertPtr(pCache);
4867	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4868	{
4869	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4870	AssertRC(rc);
4871	}
4872	pCache->Write.cValidEntries = 0;
4873	}
4874
4875
4876	/**
4877	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4878	*
4879	* @param pVCpu Pointer to the VMCPU.
4880	* @param pCache Pointer to the VMCS cache.
4881	*
4882	* @remarks No-long-jump zone!!!
4883	*/
4884	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4885	{
4886	AssertPtr(pCache);
4887	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4888	{
4889	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4890	AssertRC(rc);
4891	}
4892	}
4893	#endif
4894	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4895
4896
4897	/**
4898	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4899	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4900	* timer.
4901	*
4902	* @returns VBox status code.
4903	* @param pVCpu Pointer to the VMCPU.
4904	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4905	* out-of-sync. Make sure to update the required fields
4906	* before using them.
4907	* @remarks No-long-jump zone!!!
4908	*/
4909	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4910	{
4911	int rc = VERR_INTERNAL_ERROR_5;
4912	bool fOffsettedTsc = false;
4913	PVM pVM = pVCpu->CTX_SUFF(pVM);
4914	if (pVM->hm.s.vmx.fUsePreemptTimer)
4915	{
4916	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4917
4918	/* Make sure the returned values have sane upper and lower boundaries. */
4919	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4920	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4921	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4922	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4923
4924	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4925	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4926	}
4927	else
4928	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4929
4930	if (fOffsettedTsc)
4931	{
4932	uint64_t u64CurTSC = ASMReadTSC();
4933	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4934	{
4935	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4936	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4937
4938	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4939	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4940	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4941	}
4942	else
4943	{
4944	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4945	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4946	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4947	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4948	}
4949	}
4950	else
4951	{
4952	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4953	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4954	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4955	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4956	}
4957	}
4958
4959
4960	/**
4961	* Determines if an exception is a contributory exception. Contributory
4962	* exceptions are ones which can cause double-faults. Page-fault is
4963	* intentionally not included here as it's a conditional contributory exception.
4964	*
4965	* @returns true if the exception is contributory, false otherwise.
4966	* @param uVector The exception vector.
4967	*/
4968	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4969	{
4970	switch (uVector)
4971	{
4972	case X86_XCPT_GP:
4973	case X86_XCPT_SS:
4974	case X86_XCPT_NP:
4975	case X86_XCPT_TS:
4976	case X86_XCPT_DE:
4977	return true;
4978	default:
4979	break;
4980	}
4981	return false;
4982	}
4983
4984
4985	/**
4986	* Sets an event as a pending event to be injected into the guest.
4987	*
4988	* @param pVCpu Pointer to the VMCPU.
4989	* @param u32IntInfo The VM-entry interruption-information field.
4990	* @param cbInstr The VM-entry instruction length in bytes (for software
4991	* interrupts, exceptions and privileged software
4992	* exceptions).
4993	* @param u32ErrCode The VM-entry exception error code.
4994	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4995	* page-fault.
4996	*
4997	* @remarks Statistics counter assumes this is a guest event being injected or
4998	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4999	* always incremented.
5000	*/
5001	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5002	RTGCUINTPTR GCPtrFaultAddress)
5003	{
5004	Assert(!pVCpu->hm.s.Event.fPending);
5005	pVCpu->hm.s.Event.fPending = true;
5006	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5007	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5008	pVCpu->hm.s.Event.cbInstr = cbInstr;
5009	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5010
5011	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5012	}
5013
5014
5015	/**
5016	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5017	*
5018	* @param pVCpu Pointer to the VMCPU.
5019	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5020	* out-of-sync. Make sure to update the required fields
5021	* before using them.
5022	*/
5023	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5024	{
5025	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5026	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5027	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5028	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5029	}
5030
5031
5032	/**
5033	* Handle a condition that occurred while delivering an event through the guest
5034	* IDT.
5035	*
5036	* @returns VBox status code (informational error codes included).
5037	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5038	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5039	* continue execution of the guest which will delivery the #DF.
5040	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5041	*
5042	* @param pVCpu Pointer to the VMCPU.
5043	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5044	* out-of-sync. Make sure to update the required fields
5045	* before using them.
5046	* @param pVmxTransient Pointer to the VMX transient structure.
5047	*
5048	* @remarks No-long-jump zone!!!
5049	*/
5050	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5051	{
5052	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5053	AssertRCReturn(rc, rc);
5054	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5055	{
5056	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5057	AssertRCReturn(rc, rc);
5058
5059	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5060	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5061	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5062
5063	typedef enum
5064	{
5065	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5066	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5067	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5068	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5069	} VMXREFLECTXCPT;
5070
5071	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5072	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5073	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5074	{
5075	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5076	{
5077	enmReflect = VMXREFLECTXCPT_XCPT;
5078	#ifdef VBOX_STRICT
5079	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5080	&& uExitVector == X86_XCPT_PF)
5081	{
5082	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5083	}
5084	#endif
5085	if ( uExitVector == X86_XCPT_PF
5086	&& uIdtVector == X86_XCPT_PF)
5087	{
5088	pVmxTransient->fVectoringPF = true;
5089	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5090	}
5091	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5092	&& hmR0VmxIsContributoryXcpt(uExitVector)
5093	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5094	\|\| uIdtVector == X86_XCPT_PF))
5095	{
5096	enmReflect = VMXREFLECTXCPT_DF;
5097	}
5098	else if (uIdtVector == X86_XCPT_DF)
5099	enmReflect = VMXREFLECTXCPT_TF;
5100	}
5101	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5102	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5103	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5104	{
5105	/*
5106	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5107	* (whatever they are) as they reoccur when restarting the instruction.
5108	*/
5109	enmReflect = VMXREFLECTXCPT_XCPT;
5110	}
5111	}
5112	else
5113	{
5114	/*
5115	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5116	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5117	* original exception to the guest after handling the VM-exit.
5118	*/
5119	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5120	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5121	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5122	{
5123	enmReflect = VMXREFLECTXCPT_XCPT;
5124	}
5125	}
5126
5127	switch (enmReflect)
5128	{
5129	case VMXREFLECTXCPT_XCPT:
5130	{
5131	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5132	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5133	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5134
5135	uint32_t u32ErrCode = 0;
5136	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5137	{
5138	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5139	AssertRCReturn(rc, rc);
5140	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5141	}
5142
5143	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5144	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5145	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5146	rc = VINF_SUCCESS;
5147	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5148	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5149
5150	break;
5151	}
5152
5153	case VMXREFLECTXCPT_DF:
5154	{
5155	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5156	rc = VINF_HM_DOUBLE_FAULT;
5157	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5158	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5159
5160	break;
5161	}
5162
5163	case VMXREFLECTXCPT_TF:
5164	{
5165	rc = VINF_EM_RESET;
5166	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5167	uExitVector));
5168	break;
5169	}
5170
5171	default:
5172	Assert(rc == VINF_SUCCESS);
5173	break;
5174	}
5175	}
5176	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5177	return rc;
5178	}
5179
5180
5181	/**
5182	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5183	*
5184	* @returns VBox status code.
5185	* @param pVCpu Pointer to the VMCPU.
5186	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5187	* out-of-sync. Make sure to update the required fields
5188	* before using them.
5189	*
5190	* @remarks No-long-jump zone!!!
5191	*/
5192	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5193	{
5194	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5195	{
5196	uint32_t uVal = 0;
5197	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5198	AssertRCReturn(rc, rc);
5199
5200	uint32_t uShadow = 0;
5201	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5202	AssertRCReturn(rc, rc);
5203
5204	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5205	CPUMSetGuestCR0(pVCpu, uVal);
5206	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5207	}
5208	return VINF_SUCCESS;
5209	}
5210
5211
5212	/**
5213	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5214	*
5215	* @returns VBox status code.
5216	* @param pVCpu Pointer to the VMCPU.
5217	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5218	* out-of-sync. Make sure to update the required fields
5219	* before using them.
5220	*
5221	* @remarks No-long-jump zone!!!
5222	*/
5223	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5224	{
5225	int rc = VINF_SUCCESS;
5226	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5227	{
5228	uint32_t uVal = 0;
5229	uint32_t uShadow = 0;
5230	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5231	AssertRCReturn(rc, rc);
5232	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5233	AssertRCReturn(rc, rc);
5234
5235	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5236	CPUMSetGuestCR4(pVCpu, uVal);
5237	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5238	}
5239	return rc;
5240	}
5241
5242
5243	/**
5244	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5245	*
5246	* @returns VBox status code.
5247	* @param pVCpu Pointer to the VMCPU.
5248	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5249	* out-of-sync. Make sure to update the required fields
5250	* before using them.
5251	*
5252	* @remarks No-long-jump zone!!!
5253	*/
5254	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5255	{
5256	int rc = VINF_SUCCESS;
5257	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5258	{
5259	uint64_t u64Val = 0;
5260	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5261	AssertRCReturn(rc, rc);
5262
5263	pMixedCtx->rip = u64Val;
5264	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5265	}
5266	return rc;
5267	}
5268
5269
5270	/**
5271	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5272	*
5273	* @returns VBox status code.
5274	* @param pVCpu Pointer to the VMCPU.
5275	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5276	* out-of-sync. Make sure to update the required fields
5277	* before using them.
5278	*
5279	* @remarks No-long-jump zone!!!
5280	*/
5281	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5282	{
5283	int rc = VINF_SUCCESS;
5284	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5285	{
5286	uint64_t u64Val = 0;
5287	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5288	AssertRCReturn(rc, rc);
5289
5290	pMixedCtx->rsp = u64Val;
5291	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5292	}
5293	return rc;
5294	}
5295
5296
5297	/**
5298	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5299	*
5300	* @returns VBox status code.
5301	* @param pVCpu Pointer to the VMCPU.
5302	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5303	* out-of-sync. Make sure to update the required fields
5304	* before using them.
5305	*
5306	* @remarks No-long-jump zone!!!
5307	*/
5308	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5309	{
5310	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5311	{
5312	uint32_t uVal = 0;
5313	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5314	AssertRCReturn(rc, rc);
5315
5316	pMixedCtx->eflags.u32 = uVal;
5317	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5318	{
5319	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5320	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5321
5322	pMixedCtx->eflags.Bits.u1VM = 0;
5323	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5324	}
5325
5326	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5327	}
5328	return VINF_SUCCESS;
5329	}
5330
5331
5332	/**
5333	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5334	* guest-CPU context.
5335	*/
5336	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5337	{
5338	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5339	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5340	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5341	return rc;
5342	}
5343
5344
5345	/**
5346	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5347	* from the guest-state area in the VMCS.
5348	*
5349	* @param pVCpu Pointer to the VMCPU.
5350	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5351	* out-of-sync. Make sure to update the required fields
5352	* before using them.
5353	*
5354	* @remarks No-long-jump zone!!!
5355	*/
5356	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5357	{
5358	uint32_t uIntrState = 0;
5359	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5360	AssertRC(rc);
5361
5362	if (!uIntrState)
5363	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5364	else
5365	{
5366	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5367	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5368	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5369	AssertRC(rc);
5370	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5371	AssertRC(rc);
5372
5373	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5374	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5375	}
5376	}
5377
5378
5379	/**
5380	* Saves the guest's activity state.
5381	*
5382	* @returns VBox status code.
5383	* @param pVCpu Pointer to the VMCPU.
5384	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5385	* out-of-sync. Make sure to update the required fields
5386	* before using them.
5387	*
5388	* @remarks No-long-jump zone!!!
5389	*/
5390	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5391	{
5392	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5393	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5394	return VINF_SUCCESS;
5395	}
5396
5397
5398	/**
5399	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5400	* the current VMCS into the guest-CPU context.
5401	*
5402	* @returns VBox status code.
5403	* @param pVCpu Pointer to the VMCPU.
5404	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5405	* out-of-sync. Make sure to update the required fields
5406	* before using them.
5407	*
5408	* @remarks No-long-jump zone!!!
5409	*/
5410	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5411	{
5412	int rc = VINF_SUCCESS;
5413	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5414	{
5415	uint32_t u32Val = 0;
5416	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5417	pMixedCtx->SysEnter.cs = u32Val;
5418	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5419	}
5420
5421	uint64_t u64Val = 0;
5422	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5423	{
5424	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5425	pMixedCtx->SysEnter.eip = u64Val;
5426	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5427	}
5428	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5429	{
5430	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5431	pMixedCtx->SysEnter.esp = u64Val;
5432	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5433	}
5434	return rc;
5435	}
5436
5437
5438	/**
5439	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5440	* context.
5441	*
5442	* @returns VBox status code.
5443	* @param pVCpu Pointer to the VMCPU.
5444	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5445	* out-of-sync. Make sure to update the required fields
5446	* before using them.
5447	*
5448	* @remarks No-long-jump zone!!!
5449	*/
5450	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5451	{
5452	int rc = VINF_SUCCESS;
5453	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5454	{
5455	uint64_t u64Val = 0;
5456	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5457	pMixedCtx->fs.u64Base = u64Val;
5458	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5459	}
5460	return rc;
5461	}
5462
5463
5464	/**
5465	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5466	* context.
5467	*
5468	* @returns VBox status code.
5469	* @param pVCpu Pointer to the VMCPU.
5470	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5471	* out-of-sync. Make sure to update the required fields
5472	* before using them.
5473	*
5474	* @remarks No-long-jump zone!!!
5475	*/
5476	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5477	{
5478	int rc = VINF_SUCCESS;
5479	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5480	{
5481	uint64_t u64Val = 0;
5482	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5483	pMixedCtx->gs.u64Base = u64Val;
5484	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5485	}
5486	return rc;
5487	}
5488
5489
5490	/**
5491	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5492	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5493	* and TSC_AUX.
5494	*
5495	* @returns VBox status code.
5496	* @param pVCpu Pointer to the VMCPU.
5497	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5498	* out-of-sync. Make sure to update the required fields
5499	* before using them.
5500	*
5501	* @remarks No-long-jump zone!!!
5502	*/
5503	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5504	{
5505	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5506	return VINF_SUCCESS;
5507
5508	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5509	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5510	{
5511	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5512	pMsr += i;
5513	switch (pMsr->u32Msr)
5514	{
5515	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5516	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5517	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5518	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5519	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5520	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5521	default:
5522	{
5523	AssertFailed();
5524	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5525	}
5526	}
5527	}
5528	#endif
5529
5530	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5531	return VINF_SUCCESS;
5532	}
5533
5534
5535	/**
5536	* Saves the guest control registers from the current VMCS into the guest-CPU
5537	* context.
5538	*
5539	* @returns VBox status code.
5540	* @param pVCpu Pointer to the VMCPU.
5541	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5542	* out-of-sync. Make sure to update the required fields
5543	* before using them.
5544	*
5545	* @remarks No-long-jump zone!!!
5546	*/
5547	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5548	{
5549	/* Guest CR0. Guest FPU. */
5550	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5551	AssertRCReturn(rc, rc);
5552
5553	/* Guest CR4. */
5554	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5555	AssertRCReturn(rc, rc);
5556
5557	/* Guest CR2 - updated always during the world-switch or in #PF. */
5558	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5559	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5560	{
5561	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5562	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5563
5564	PVM pVM = pVCpu->CTX_SUFF(pVM);
5565	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5566	\|\| ( pVM->hm.s.fNestedPaging
5567	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5568	{
5569	uint64_t u64Val = 0;
5570	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5571	if (pMixedCtx->cr3 != u64Val)
5572	{
5573	CPUMSetGuestCR3(pVCpu, u64Val);
5574	if (VMMRZCallRing3IsEnabled(pVCpu))
5575	{
5576	PGMUpdateCR3(pVCpu, u64Val);
5577	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5578	}
5579	else
5580	{
5581	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5582	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5583	}
5584	}
5585
5586	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5587	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5588	{
5589	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5590	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5591	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5592	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5593
5594	if (VMMRZCallRing3IsEnabled(pVCpu))
5595	{
5596	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5597	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5598	}
5599	else
5600	{
5601	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5602	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5603	}
5604	}
5605	}
5606
5607	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5608	}
5609
5610	/*
5611	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5612	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5613	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5614	*
5615	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5616	*/
5617	if (VMMRZCallRing3IsEnabled(pVCpu))
5618	{
5619	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5620	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5621
5622	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5623	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5624
5625	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5626	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5627	}
5628
5629	return rc;
5630	}
5631
5632
5633	/**
5634	* Reads a guest segment register from the current VMCS into the guest-CPU
5635	* context.
5636	*
5637	* @returns VBox status code.
5638	* @param pVCpu Pointer to the VMCPU.
5639	* @param idxSel Index of the selector in the VMCS.
5640	* @param idxLimit Index of the segment limit in the VMCS.
5641	* @param idxBase Index of the segment base in the VMCS.
5642	* @param idxAccess Index of the access rights of the segment in the VMCS.
5643	* @param pSelReg Pointer to the segment selector.
5644	*
5645	* @remarks No-long-jump zone!!!
5646	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5647	* macro as that takes care of whether to read from the VMCS cache or
5648	* not.
5649	*/
5650	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5651	PCPUMSELREG pSelReg)
5652	{
5653	uint32_t u32Val = 0;
5654	int rc = VMXReadVmcs32(idxSel, &u32Val);
5655	AssertRCReturn(rc, rc);
5656	pSelReg->Sel = (uint16_t)u32Val;
5657	pSelReg->ValidSel = (uint16_t)u32Val;
5658	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5659
5660	rc = VMXReadVmcs32(idxLimit, &u32Val);
5661	AssertRCReturn(rc, rc);
5662	pSelReg->u32Limit = u32Val;
5663
5664	uint64_t u64Val = 0;
5665	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5666	AssertRCReturn(rc, rc);
5667	pSelReg->u64Base = u64Val;
5668
5669	rc = VMXReadVmcs32(idxAccess, &u32Val);
5670	AssertRCReturn(rc, rc);
5671	pSelReg->Attr.u = u32Val;
5672
5673	/*
5674	* If VT-x marks the segment as unusable, most other bits remain undefined:
5675	* - For CS the L, D and G bits have meaning.
5676	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5677	* - For the remaining data segments no bits are defined.
5678	*
5679	* The present bit and the unusable bit has been observed to be set at the
5680	* same time (the selector was supposed to invalid as we started executing
5681	* a V8086 interrupt in ring-0).
5682	*
5683	* What should be important for the rest of the VBox code that the P bit is
5684	* cleared. Some of the other VBox code recognizes the unusable bit, but
5685	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5686	* safe side here, we'll strip off P and other bits we don't care about. If
5687	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5688	*
5689	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5690	*/
5691	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5692	{
5693	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5694
5695	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5696	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5697	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5698
5699	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5700	#ifdef DEBUG_bird
5701	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5702	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5703	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5704	#endif
5705	}
5706	return VINF_SUCCESS;
5707	}
5708
5709
5710	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5711	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5712	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5713	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5714	#else
5715	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5716	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5717	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5718	#endif
5719
5720
5721	/**
5722	* Saves the guest segment registers from the current VMCS into the guest-CPU
5723	* context.
5724	*
5725	* @returns VBox status code.
5726	* @param pVCpu Pointer to the VMCPU.
5727	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5728	* out-of-sync. Make sure to update the required fields
5729	* before using them.
5730	*
5731	* @remarks No-long-jump zone!!!
5732	*/
5733	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5734	{
5735	/* Guest segment registers. */
5736	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5737	{
5738	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5739	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5740	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5741	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5742	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5743	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5744	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5745
5746	/* Restore segment attributes for real-on-v86 mode hack. */
5747	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5748	{
5749	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5750	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5751	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5752	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5753	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5754	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5755	}
5756	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5757	}
5758
5759	return VINF_SUCCESS;
5760	}
5761
5762
5763	/**
5764	* Saves the guest descriptor table registers and task register from the current
5765	* VMCS into the guest-CPU context.
5766	*
5767	* @returns VBox status code.
5768	* @param pVCpu Pointer to the VMCPU.
5769	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5770	* out-of-sync. Make sure to update the required fields
5771	* before using them.
5772	*
5773	* @remarks No-long-jump zone!!!
5774	*/
5775	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5776	{
5777	int rc = VINF_SUCCESS;
5778
5779	/* Guest LDTR. */
5780	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5781	{
5782	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5783	AssertRCReturn(rc, rc);
5784	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5785	}
5786
5787	/* Guest GDTR. */
5788	uint64_t u64Val = 0;
5789	uint32_t u32Val = 0;
5790	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5791	{
5792	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5793	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5794	pMixedCtx->gdtr.pGdt = u64Val;
5795	pMixedCtx->gdtr.cbGdt = u32Val;
5796	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5797	}
5798
5799	/* Guest IDTR. */
5800	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5801	{
5802	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5803	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5804	pMixedCtx->idtr.pIdt = u64Val;
5805	pMixedCtx->idtr.cbIdt = u32Val;
5806	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5807	}
5808
5809	/* Guest TR. */
5810	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5811	{
5812	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5813	AssertRCReturn(rc, rc);
5814
5815	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5816	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5817	{
5818	rc = VMXLOCAL_READ_SEG(TR, tr);
5819	AssertRCReturn(rc, rc);
5820	}
5821	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5822	}
5823	return rc;
5824	}
5825
5826	#undef VMXLOCAL_READ_SEG
5827
5828
5829	/**
5830	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5831	* context.
5832	*
5833	* @returns VBox status code.
5834	* @param pVCpu Pointer to the VMCPU.
5835	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5836	* out-of-sync. Make sure to update the required fields
5837	* before using them.
5838	*
5839	* @remarks No-long-jump zone!!!
5840	*/
5841	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5842	{
5843	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5844	{
5845	if (!pVCpu->hm.s.fUsingHyperDR7)
5846	{
5847	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5848	uint32_t u32Val;
5849	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5850	pMixedCtx->dr[7] = u32Val;
5851	}
5852
5853	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5854	}
5855	return VINF_SUCCESS;
5856	}
5857
5858
5859	/**
5860	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5861	*
5862	* @returns VBox status code.
5863	* @param pVCpu Pointer to the VMCPU.
5864	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5865	* out-of-sync. Make sure to update the required fields
5866	* before using them.
5867	*
5868	* @remarks No-long-jump zone!!!
5869	*/
5870	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5871	{
5872	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5873	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5874	return VINF_SUCCESS;
5875	}
5876
5877
5878	/**
5879	* Saves the entire guest state from the currently active VMCS into the
5880	* guest-CPU context. This essentially VMREADs all guest-data.
5881	*
5882	* @returns VBox status code.
5883	* @param pVCpu Pointer to the VMCPU.
5884	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5885	* out-of-sync. Make sure to update the required fields
5886	* before using them.
5887	*/
5888	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5889	{
5890	Assert(pVCpu);
5891	Assert(pMixedCtx);
5892
5893	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5894	return VINF_SUCCESS;
5895
5896	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5897	again on the ring-3 callback path, there is no real need to. */
5898	if (VMMRZCallRing3IsEnabled(pVCpu))
5899	VMMR0LogFlushDisable(pVCpu);
5900	else
5901	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5902	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5903
5904	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5905	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5906
5907	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5908	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5909
5910	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5911	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5912
5913	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5914	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5915
5916	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5917	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5918
5919	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5920	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5921
5922	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5923	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5924
5925	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5926	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5927
5928	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5929	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5930
5931	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5932	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5933
5934	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5935	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5936
5937	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5938	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5939
5940	if (VMMRZCallRing3IsEnabled(pVCpu))
5941	VMMR0LogFlushEnable(pVCpu);
5942
5943	return rc;
5944	}
5945
5946
5947	/**
5948	* Check per-VM and per-VCPU force flag actions that require us to go back to
5949	* ring-3 for one reason or another.
5950	*
5951	* @returns VBox status code (information status code included).
5952	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5953	* ring-3.
5954	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5955	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5956	* interrupts)
5957	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5958	* all EMTs to be in ring-3.
5959	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5960	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5961	* to the EM loop.
5962	*
5963	* @param pVM Pointer to the VM.
5964	* @param pVCpu Pointer to the VMCPU.
5965	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5966	* out-of-sync. Make sure to update the required fields
5967	* before using them.
5968	*/
5969	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5970	{
5971	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5972
5973	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5974	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5975	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5976	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5977	{
5978	/* We need the control registers now, make sure the guest-CPU context is updated. */
5979	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5980	AssertRCReturn(rc3, rc3);
5981
5982	/* Pending HM CR3 sync. */
5983	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5984	{
5985	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5986	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
5987	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5988	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5989	}
5990
5991	/* Pending HM PAE PDPEs. */
5992	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5993	{
5994	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5995	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5996	}
5997
5998	/* Pending PGM C3 sync. */
5999	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6000	{
6001	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6002	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6003	if (rc2 != VINF_SUCCESS)
6004	{
6005	AssertRC(rc2);
6006	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6007	return rc2;
6008	}
6009	}
6010
6011	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6012	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6013	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6014	{
6015	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6016	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6017	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6018	return rc2;
6019	}
6020
6021	/* Pending VM request packets, such as hardware interrupts. */
6022	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6023	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6024	{
6025	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6026	return VINF_EM_PENDING_REQUEST;
6027	}
6028
6029	/* Pending PGM pool flushes. */
6030	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6031	{
6032	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6033	return VINF_PGM_POOL_FLUSH_PENDING;
6034	}
6035
6036	/* Pending DMA requests. */
6037	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6038	{
6039	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6040	return VINF_EM_RAW_TO_R3;
6041	}
6042	}
6043
6044	return VINF_SUCCESS;
6045	}
6046
6047
6048	/**
6049	* Converts any TRPM trap into a pending HM event. This is typically used when
6050	* entering from ring-3 (not longjmp returns).
6051	*
6052	* @param pVCpu Pointer to the VMCPU.
6053	*/
6054	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6055	{
6056	Assert(TRPMHasTrap(pVCpu));
6057	Assert(!pVCpu->hm.s.Event.fPending);
6058
6059	uint8_t uVector;
6060	TRPMEVENT enmTrpmEvent;
6061	RTGCUINT uErrCode;
6062	RTGCUINTPTR GCPtrFaultAddress;
6063	uint8_t cbInstr;
6064
6065	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6066	AssertRC(rc);
6067
6068	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6069	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6070	if (enmTrpmEvent == TRPM_TRAP)
6071	{
6072	switch (uVector)
6073	{
6074	case X86_XCPT_BP:
6075	case X86_XCPT_OF:
6076	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6077	break;
6078
6079	case X86_XCPT_PF:
6080	case X86_XCPT_DF:
6081	case X86_XCPT_TS:
6082	case X86_XCPT_NP:
6083	case X86_XCPT_SS:
6084	case X86_XCPT_GP:
6085	case X86_XCPT_AC:
6086	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6087	/* no break! */
6088	default:
6089	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6090	break;
6091	}
6092	}
6093	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6094	{
6095	if (uVector == X86_XCPT_NMI)
6096	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6097	else
6098	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6099	}
6100	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6101	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6102	else
6103	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6104
6105	rc = TRPMResetTrap(pVCpu);
6106	AssertRC(rc);
6107	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6108	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6109
6110	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6111	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6112	}
6113
6114
6115	/**
6116	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6117	* VT-x to execute any instruction.
6118	*
6119	* @param pvCpu Pointer to the VMCPU.
6120	*/
6121	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6122	{
6123	Assert(pVCpu->hm.s.Event.fPending);
6124
6125	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6126	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6127	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6128	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6129
6130	/* If a trap was already pending, we did something wrong! */
6131	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6132
6133	TRPMEVENT enmTrapType;
6134	switch (uVectorType)
6135	{
6136	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6137	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6138	enmTrapType = TRPM_HARDWARE_INT;
6139	break;
6140
6141	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6142	enmTrapType = TRPM_SOFTWARE_INT;
6143	break;
6144
6145	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6146	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6147	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6148	enmTrapType = TRPM_TRAP;
6149	break;
6150
6151	default:
6152	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6153	enmTrapType = TRPM_32BIT_HACK;
6154	break;
6155	}
6156
6157	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6158
6159	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6160	AssertRC(rc);
6161
6162	if (fErrorCodeValid)
6163	TRPMSetErrorCode(pVCpu, uErrorCode);
6164
6165	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6166	&& uVector == X86_XCPT_PF)
6167	{
6168	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6169	}
6170	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6171	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6172	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6173	{
6174	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6175	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6176	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6177	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6178	}
6179	pVCpu->hm.s.Event.fPending = false;
6180	}
6181
6182
6183	/**
6184	* Does the necessary state syncing before returning to ring-3 for any reason
6185	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6186	*
6187	* @returns VBox status code.
6188	* @param pVM Pointer to the VM.
6189	* @param pVCpu Pointer to the VMCPU.
6190	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6191	* be out-of-sync. Make sure to update the required
6192	* fields before using them.
6193	* @param fSaveGuestState Whether to save the guest state or not.
6194	*
6195	* @remarks No-long-jmp zone!!!
6196	*/
6197	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6198	{
6199	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6200	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6201
6202	RTCPUID idCpu = RTMpCpuId();
6203	Log4Func(("HostCpuId=%u\n", idCpu));
6204
6205	/* Save the guest state if necessary. */
6206	if ( fSaveGuestState
6207	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6208	{
6209	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6210	AssertRCReturn(rc, rc);
6211	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6212	}
6213
6214	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6215	if (CPUMIsGuestFPUStateActive(pVCpu))
6216	{
6217	/* We shouldn't reload CR0 without saving it first. */
6218	if (!fSaveGuestState)
6219	{
6220	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6221	AssertRCReturn(rc, rc);
6222	}
6223	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6224	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6225	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6226	}
6227
6228	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6229	#ifdef VBOX_STRICT
6230	if (CPUMIsHyperDebugStateActive(pVCpu))
6231	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6232	#endif
6233	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6234	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6235	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6236	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6237
6238	#if HC_ARCH_BITS == 64
6239	/* Restore host-state bits that VT-x only restores partially. */
6240	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6241	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6242	{
6243	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6244	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6245	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6246	}
6247	#endif
6248
6249	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6250	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6251	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6252	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6253	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6254	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6255	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6256	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6257
6258	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6259
6260	/** @todo This kinda defeats the purpose of having preemption hooks.
6261	* The problem is, deregistering the hooks should be moved to a place that
6262	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6263	* context.
6264	*/
6265	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6266	{
6267	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6268	AssertRCReturn(rc, rc);
6269
6270	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6271	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6272	}
6273	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6274	NOREF(idCpu);
6275
6276	return VINF_SUCCESS;
6277	}
6278
6279
6280	/**
6281	* Leaves the VT-x session.
6282	*
6283	* @returns VBox status code.
6284	* @param pVM Pointer to the VM.
6285	* @param pVCpu Pointer to the VMCPU.
6286	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6287	* out-of-sync. Make sure to update the required fields
6288	* before using them.
6289	*
6290	* @remarks No-long-jmp zone!!!
6291	*/
6292	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6293	{
6294	HM_DISABLE_PREEMPT_IF_NEEDED();
6295	HMVMX_ASSERT_CPU_SAFE();
6296	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6297	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6298
6299	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6300	and done this from the VMXR0ThreadCtxCallback(). */
6301	if (!pVCpu->hm.s.fLeaveDone)
6302	{
6303	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6304	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6305	pVCpu->hm.s.fLeaveDone = true;
6306	}
6307
6308	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6309	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6310	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6311	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6312	VMMR0ThreadCtxHooksDeregister(pVCpu);
6313
6314	/* Leave HM context. This takes care of local init (term). */
6315	int rc = HMR0LeaveCpu(pVCpu);
6316
6317	HM_RESTORE_PREEMPT_IF_NEEDED();
6318
6319	return rc;
6320	}
6321
6322
6323	/**
6324	* Does the necessary state syncing before doing a longjmp to ring-3.
6325	*
6326	* @returns VBox status code.
6327	* @param pVM Pointer to the VM.
6328	* @param pVCpu Pointer to the VMCPU.
6329	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6330	* out-of-sync. Make sure to update the required fields
6331	* before using them.
6332	*
6333	* @remarks No-long-jmp zone!!!
6334	*/
6335	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6336	{
6337	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6338	}
6339
6340
6341	/**
6342	* Take necessary actions before going back to ring-3.
6343	*
6344	* An action requires us to go back to ring-3. This function does the necessary
6345	* steps before we can safely return to ring-3. This is not the same as longjmps
6346	* to ring-3, this is voluntary and prepares the guest so it may continue
6347	* executing outside HM (recompiler/IEM).
6348	*
6349	* @returns VBox status code.
6350	* @param pVM Pointer to the VM.
6351	* @param pVCpu Pointer to the VMCPU.
6352	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6353	* out-of-sync. Make sure to update the required fields
6354	* before using them.
6355	* @param rcExit The reason for exiting to ring-3. Can be
6356	* VINF_VMM_UNKNOWN_RING3_CALL.
6357	*/
6358	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6359	{
6360	Assert(pVM);
6361	Assert(pVCpu);
6362	Assert(pMixedCtx);
6363	HMVMX_ASSERT_PREEMPT_SAFE();
6364
6365	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6366	{
6367	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6368	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6369	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6370	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6371	}
6372
6373	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6374	VMMRZCallRing3Disable(pVCpu);
6375	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6376
6377	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6378	if (pVCpu->hm.s.Event.fPending)
6379	{
6380	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6381	Assert(!pVCpu->hm.s.Event.fPending);
6382	}
6383
6384	/* Save guest state and restore host state bits. */
6385	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6386	AssertRCReturn(rc, rc);
6387	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6388
6389	/* Sync recompiler state. */
6390	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6391	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6392	\| CPUM_CHANGED_LDTR
6393	\| CPUM_CHANGED_GDTR
6394	\| CPUM_CHANGED_IDTR
6395	\| CPUM_CHANGED_TR
6396	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6397	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6398	if ( pVM->hm.s.fNestedPaging
6399	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6400	{
6401	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6402	}
6403
6404	/*
6405	* Clear the X86_EFL_TF if necessary.
6406	*/
6407	if (pVCpu->hm.s.fClearTrapFlag)
6408	{
6409	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6410	pMixedCtx->eflags.Bits.u1TF = 0;
6411	pVCpu->hm.s.fClearTrapFlag = false;
6412	}
6413	/** @todo there seems to be issues with the resume flag when the monitor trap
6414	* flag is pending without being used. Seen early in bios init when
6415	* accessing APIC page in prot mode. */
6416
6417	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6418	if (rcExit != VINF_EM_RAW_INTERRUPT)
6419	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6420
6421	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6422
6423	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6424	VMMRZCallRing3RemoveNotification(pVCpu);
6425	VMMRZCallRing3Enable(pVCpu);
6426
6427	return rc;
6428	}
6429
6430
6431	/**
6432	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6433	* longjump to ring-3 and possibly get preempted.
6434	*
6435	* @returns VBox status code.
6436	* @param pVCpu Pointer to the VMCPU.
6437	* @param enmOperation The operation causing the ring-3 longjump.
6438	* @param pvUser Opaque pointer to the guest-CPU context. The data
6439	* may be out-of-sync. Make sure to update the required
6440	* fields before using them.
6441	*/
6442	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6443	{
6444	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6445	{
6446	VMMRZCallRing3RemoveNotification(pVCpu);
6447	HM_DISABLE_PREEMPT_IF_NEEDED();
6448
6449	/* If anything here asserts or fails, good luck. */
6450	if (CPUMIsGuestFPUStateActive(pVCpu))
6451	CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6452
6453	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6454
6455	#if HC_ARCH_BITS == 64
6456	/* Restore host-state bits that VT-x only restores partially. */
6457	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6458	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6459	{
6460	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6461	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6462	}
6463	#endif
6464	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6465	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6466	{
6467	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6468	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6469	}
6470
6471	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6472	VMMR0ThreadCtxHooksDeregister(pVCpu);
6473
6474	HMR0LeaveCpu(pVCpu);
6475	HM_RESTORE_PREEMPT_IF_NEEDED();
6476	return VINF_SUCCESS;
6477	}
6478
6479	Assert(pVCpu);
6480	Assert(pvUser);
6481	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6482	HMVMX_ASSERT_PREEMPT_SAFE();
6483
6484	VMMRZCallRing3Disable(pVCpu);
6485	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6486
6487	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6488	enmOperation));
6489
6490	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6491	AssertRCReturn(rc, rc);
6492
6493	VMMRZCallRing3Enable(pVCpu);
6494	return VINF_SUCCESS;
6495	}
6496
6497
6498	/**
6499	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6500	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6501	*
6502	* @param pVCpu Pointer to the VMCPU.
6503	*/
6504	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6505	{
6506	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6507	{
6508	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6509	{
6510	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6511	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6512	AssertRC(rc);
6513	}
6514	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6515	}
6516
6517
6518	/**
6519	* Evaluates the event to be delivered to the guest and sets it as the pending
6520	* event.
6521	*
6522	* @param pVCpu Pointer to the VMCPU.
6523	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6524	* out-of-sync. Make sure to update the required fields
6525	* before using them.
6526	*/
6527	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6528	{
6529	Assert(!pVCpu->hm.s.Event.fPending);
6530
6531	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6532	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6533	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6534	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6535
6536	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6537	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6538	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6539	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6540	Assert(!TRPMHasTrap(pVCpu));
6541
6542	/** @todo SMI. SMIs take priority over NMIs. */
6543	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6544	{
6545	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6546	if ( !fBlockMovSS
6547	&& !fBlockSti)
6548	{
6549	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6550	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6551	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6552	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6553
6554	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6555	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6556	}
6557	else
6558	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6559	}
6560	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6561	&& !pVCpu->hm.s.fSingleInstruction)
6562	{
6563	/*
6564	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6565	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6566	* evaluated here and not set as pending, solely based on the force-flags.
6567	*/
6568	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6569	AssertRC(rc);
6570	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6571	if ( !fBlockInt
6572	&& !fBlockSti
6573	&& !fBlockMovSS)
6574	{
6575	uint8_t u8Interrupt;
6576	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6577	if (RT_SUCCESS(rc))
6578	{
6579	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6580	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6581	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6582
6583	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6584	}
6585	else
6586	{
6587	/** @todo Does this actually happen? If not turn it into an assertion. */
6588	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6589	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6590	}
6591	}
6592	else
6593	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6594	}
6595	}
6596
6597
6598	/**
6599	* Injects any pending events into the guest if the guest is in a state to
6600	* receive them.
6601	*
6602	* @returns VBox status code (informational status codes included).
6603	* @param pVCpu Pointer to the VMCPU.
6604	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6605	* out-of-sync. Make sure to update the required fields
6606	* before using them.
6607	*/
6608	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6609	{
6610	HMVMX_ASSERT_PREEMPT_SAFE();
6611	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6612
6613	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6614	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6615	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6616	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6617
6618	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6619	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6620	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6621	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6622	Assert(!TRPMHasTrap(pVCpu));
6623
6624	int rc = VINF_SUCCESS;
6625	if (pVCpu->hm.s.Event.fPending)
6626	{
6627	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
6628	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6629	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6630	{
6631	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6632	AssertRCReturn(rc, rc);
6633	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6634	Assert(!fBlockInt);
6635	Assert(!fBlockSti);
6636	Assert(!fBlockMovSS);
6637	}
6638	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6639	{
6640	Assert(!fBlockSti);
6641	Assert(!fBlockMovSS);
6642	}
6643	#endif
6644	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6645	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6646	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6647	AssertRCReturn(rc, rc);
6648
6649	/* Update the interruptibility-state as it could have been changed by
6650	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6651	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6652	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6653
6654	#ifdef VBOX_WITH_STATISTICS
6655	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6656	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6657	else
6658	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6659	#endif
6660	}
6661
6662	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6663	int rc2 = VINF_SUCCESS;
6664	if ( fBlockSti
6665	\|\| fBlockMovSS)
6666	{
6667	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6668	{
6669	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6670	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6671	{
6672	/*
6673	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6674	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6675	* See Intel spec. 27.3.4 "Saving Non-Register State".
6676	*/
6677	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6678	AssertRCReturn(rc, rc);
6679	}
6680	}
6681	else
6682	{
6683	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6684	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6685	uIntrState = 0;
6686	}
6687	}
6688
6689	/*
6690	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6691	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6692	*/
6693	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6694	AssertRC(rc2);
6695
6696	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6697	return rc;
6698	}
6699
6700
6701	/**
6702	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6703	*
6704	* @param pVCpu Pointer to the VMCPU.
6705	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6706	* out-of-sync. Make sure to update the required fields
6707	* before using them.
6708	*/
6709	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6710	{
6711	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6712	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6713	}
6714
6715
6716	/**
6717	* Injects a double-fault (#DF) exception into the VM.
6718	*
6719	* @returns VBox status code (informational status code included).
6720	* @param pVCpu Pointer to the VMCPU.
6721	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6722	* out-of-sync. Make sure to update the required fields
6723	* before using them.
6724	*/
6725	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6726	{
6727	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6728	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6729	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6730	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6731	puIntrState);
6732	}
6733
6734
6735	/**
6736	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6737	*
6738	* @param pVCpu Pointer to the VMCPU.
6739	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6740	* out-of-sync. Make sure to update the required fields
6741	* before using them.
6742	*/
6743	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6744	{
6745	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6746	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6747	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6748	}
6749
6750
6751	/**
6752	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6753	*
6754	* @param pVCpu Pointer to the VMCPU.
6755	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6756	* out-of-sync. Make sure to update the required fields
6757	* before using them.
6758	* @param cbInstr The value of RIP that is to be pushed on the guest
6759	* stack.
6760	*/
6761	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6762	{
6763	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6764	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6765	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6766	}
6767
6768
6769	/**
6770	* Injects a general-protection (#GP) fault into the VM.
6771	*
6772	* @returns VBox status code (informational status code included).
6773	* @param pVCpu Pointer to the VMCPU.
6774	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6775	* out-of-sync. Make sure to update the required fields
6776	* before using them.
6777	* @param u32ErrorCode The error code associated with the #GP.
6778	*/
6779	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6780	uint32_t *puIntrState)
6781	{
6782	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6783	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6784	if (fErrorCodeValid)
6785	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6786	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6787	puIntrState);
6788	}
6789
6790
6791	/**
6792	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6793	*
6794	* @param pVCpu Pointer to the VMCPU.
6795	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6796	* out-of-sync. Make sure to update the required fields
6797	* before using them.
6798	* @param uVector The software interrupt vector number.
6799	* @param cbInstr The value of RIP that is to be pushed on the guest
6800	* stack.
6801	*/
6802	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6803	{
6804	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6805	if ( uVector == X86_XCPT_BP
6806	\|\| uVector == X86_XCPT_OF)
6807	{
6808	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6809	}
6810	else
6811	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6812	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6813	}
6814
6815
6816	/**
6817	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6818	* stack.
6819	*
6820	* @returns VBox status code (information status code included).
6821	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6822	* @param pVM Pointer to the VM.
6823	* @param pMixedCtx Pointer to the guest-CPU context.
6824	* @param uValue The value to push to the guest stack.
6825	*/
6826	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6827	{
6828	/*
6829	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6830	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6831	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6832	*/
6833	if (pMixedCtx->sp == 1)
6834	return VINF_EM_RESET;
6835	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6836	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6837	AssertRCReturn(rc, rc);
6838	return rc;
6839	}
6840
6841
6842	/**
6843	* Injects an event into the guest upon VM-entry by updating the relevant fields
6844	* in the VM-entry area in the VMCS.
6845	*
6846	* @returns VBox status code (informational error codes included).
6847	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6848	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6849	*
6850	* @param pVCpu Pointer to the VMCPU.
6851	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6852	* be out-of-sync. Make sure to update the required
6853	* fields before using them.
6854	* @param u64IntInfo The VM-entry interruption-information field.
6855	* @param cbInstr The VM-entry instruction length in bytes (for
6856	* software interrupts, exceptions and privileged
6857	* software exceptions).
6858	* @param u32ErrCode The VM-entry exception error code.
6859	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6860	* @param puIntrState Pointer to the current guest interruptibility-state.
6861	* This interruptibility-state will be updated if
6862	* necessary. This cannot not be NULL.
6863	*
6864	* @remarks Requires CR0!
6865	* @remarks No-long-jump zone!!!
6866	*/
6867	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
6868	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6869	{
6870	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6871	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
6872	Assert(puIntrState);
6873	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
6874
6875	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
6876	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
6877
6878	#ifdef VBOX_STRICT
6879	/* Validate the error-code-valid bit for hardware exceptions. */
6880	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6881	{
6882	switch (uVector)
6883	{
6884	case X86_XCPT_PF:
6885	case X86_XCPT_DF:
6886	case X86_XCPT_TS:
6887	case X86_XCPT_NP:
6888	case X86_XCPT_SS:
6889	case X86_XCPT_GP:
6890	case X86_XCPT_AC:
6891	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
6892	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6893	/* fallthru */
6894	default:
6895	break;
6896	}
6897	}
6898	#endif
6899
6900	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6901	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6902	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6903
6904	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6905
6906	/* We require CR0 to check if the guest is in real-mode. */
6907	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6908	AssertRCReturn(rc, rc);
6909
6910	/*
6911	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6912	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6913	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6914	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6915	*/
6916	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6917	{
6918	PVM pVM = pVCpu->CTX_SUFF(pVM);
6919	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6920	{
6921	Assert(PDMVmmDevHeapIsEnabled(pVM));
6922	Assert(pVM->hm.s.vmx.pRealModeTSS);
6923
6924	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6925	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6926	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6927	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6928	AssertRCReturn(rc, rc);
6929	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6930
6931	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6932	const size_t cbIdtEntry = sizeof(X86IDTR16);
6933	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6934	{
6935	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6936	if (uVector == X86_XCPT_DF)
6937	return VINF_EM_RESET;
6938	else if (uVector == X86_XCPT_GP)
6939	{
6940	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6941	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6942	}
6943
6944	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6945	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6946	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6947	}
6948
6949	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6950	uint16_t uGuestIp = pMixedCtx->ip;
6951	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6952	{
6953	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6954	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6955	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6956	}
6957	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6958	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6959
6960	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6961	X86IDTR16 IdtEntry;
6962	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6963	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6964	AssertRCReturn(rc, rc);
6965
6966	/* Construct the stack frame for the interrupt/exception handler. */
6967	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6968	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6969	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6970	AssertRCReturn(rc, rc);
6971
6972	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6973	if (rc == VINF_SUCCESS)
6974	{
6975	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6976	pMixedCtx->rip = IdtEntry.offSel;
6977	pMixedCtx->cs.Sel = IdtEntry.uSel;
6978	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6979	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6980	&& uVector == X86_XCPT_PF)
6981	{
6982	pMixedCtx->cr2 = GCPtrFaultAddress;
6983	}
6984
6985	/* If any other guest-state bits are changed here, make sure to update
6986	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6987	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
6988	\| HM_CHANGED_GUEST_RIP
6989	\| HM_CHANGED_GUEST_RFLAGS
6990	\| HM_CHANGED_GUEST_RSP);
6991
6992	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6993	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6994	{
6995	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6996	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6997	Log4(("Clearing inhibition due to STI.\n"));
6998	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6999	}
7000	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7001
7002	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7003	it, if we are returning to ring-3 before executing guest code. */
7004	pVCpu->hm.s.Event.fPending = false;
7005	}
7006	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7007	return rc;
7008	}
7009	else
7010	{
7011	/*
7012	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7013	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7014	*/
7015	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7016	}
7017	}
7018
7019	/* Validate. */
7020	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7021	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7022	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7023
7024	/* Inject. */
7025	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7026	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7027	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7028	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7029
7030	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7031	&& uVector == X86_XCPT_PF)
7032	{
7033	pMixedCtx->cr2 = GCPtrFaultAddress;
7034	}
7035
7036	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7037	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7038
7039	AssertRCReturn(rc, rc);
7040	return rc;
7041	}
7042
7043
7044	/**
7045	* Clears the interrupt-window exiting control in the VMCS and if necessary
7046	* clears the current event in the VMCS as well.
7047	*
7048	* @returns VBox status code.
7049	* @param pVCpu Pointer to the VMCPU.
7050	*
7051	* @remarks Use this function only to clear events that have not yet been
7052	* delivered to the guest but are injected in the VMCS!
7053	* @remarks No-long-jump zone!!!
7054	*/
7055	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7056	{
7057	int rc;
7058	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7059
7060	/* Clear interrupt-window exiting control. */
7061	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7062	{
7063	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7064	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7065	AssertRC(rc);
7066	}
7067
7068	if (!pVCpu->hm.s.Event.fPending)
7069	return;
7070
7071	#ifdef VBOX_STRICT
7072	uint32_t u32EntryInfo;
7073	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7074	AssertRC(rc);
7075	Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7076	#endif
7077
7078	/* Clear the entry-interruption field (including the valid bit). */
7079	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7080	AssertRC(rc);
7081
7082	/* Clear the pending debug exception field. */
7083	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7084	AssertRC(rc);
7085	}
7086
7087
7088	/**
7089	* Enters the VT-x session.
7090	*
7091	* @returns VBox status code.
7092	* @param pVM Pointer to the VM.
7093	* @param pVCpu Pointer to the VMCPU.
7094	* @param pCpu Pointer to the CPU info struct.
7095	*/
7096	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7097	{
7098	AssertPtr(pVM);
7099	AssertPtr(pVCpu);
7100	Assert(pVM->hm.s.vmx.fSupported);
7101	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7102	NOREF(pCpu);
7103
7104	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7105	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7106
7107	#ifdef VBOX_STRICT
7108	/* Make sure we're in VMX root mode. */
7109	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7110	if (!(u32HostCR4 & X86_CR4_VMXE))
7111	{
7112	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7113	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7114	}
7115	#endif
7116
7117	/*
7118	* Load the VCPU's VMCS as the current (and active) one.
7119	*/
7120	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7121	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7122	if (RT_FAILURE(rc))
7123	return rc;
7124
7125	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7126	pVCpu->hm.s.fLeaveDone = false;
7127	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7128
7129	return VINF_SUCCESS;
7130	}
7131
7132
7133	/**
7134	* The thread-context callback (only on platforms which support it).
7135	*
7136	* @param enmEvent The thread-context event.
7137	* @param pVCpu Pointer to the VMCPU.
7138	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7139	* @thread EMT.
7140	*/
7141	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7142	{
7143	switch (enmEvent)
7144	{
7145	case RTTHREADCTXEVENT_PREEMPTING:
7146	{
7147	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7148	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7149	VMCPU_ASSERT_EMT(pVCpu);
7150
7151	PVM pVM = pVCpu->CTX_SUFF(pVM);
7152	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7153
7154	/* No longjmps (logger flushes, locks) in this fragile context. */
7155	VMMRZCallRing3Disable(pVCpu);
7156	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7157
7158	/*
7159	* Restore host-state (FPU, debug etc.)
7160	*/
7161	if (!pVCpu->hm.s.fLeaveDone)
7162	{
7163	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7164	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7165	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7166	pVCpu->hm.s.fLeaveDone = true;
7167	}
7168
7169	/* Leave HM context, takes care of local init (term). */
7170	int rc = HMR0LeaveCpu(pVCpu);
7171	AssertRC(rc); NOREF(rc);
7172
7173	/* Restore longjmp state. */
7174	VMMRZCallRing3Enable(pVCpu);
7175	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7176	break;
7177	}
7178
7179	case RTTHREADCTXEVENT_RESUMED:
7180	{
7181	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7182	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7183	VMCPU_ASSERT_EMT(pVCpu);
7184
7185	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7186	VMMRZCallRing3Disable(pVCpu);
7187	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7188
7189	/* Initialize the bare minimum state required for HM. This takes care of
7190	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7191	int rc = HMR0EnterCpu(pVCpu);
7192	AssertRC(rc);
7193	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7194
7195	/* Load the active VMCS as the current one. */
7196	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7197	{
7198	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7199	AssertRC(rc); NOREF(rc);
7200	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7201	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7202	}
7203	pVCpu->hm.s.fLeaveDone = false;
7204
7205	/* Restore longjmp state. */
7206	VMMRZCallRing3Enable(pVCpu);
7207	break;
7208	}
7209
7210	default:
7211	break;
7212	}
7213	}
7214
7215
7216	/**
7217	* Saves the host state in the VMCS host-state.
7218	* Sets up the VM-exit MSR-load area.
7219	*
7220	* The CPU state will be loaded from these fields on every successful VM-exit.
7221	*
7222	* @returns VBox status code.
7223	* @param pVM Pointer to the VM.
7224	* @param pVCpu Pointer to the VMCPU.
7225	*
7226	* @remarks No-long-jump zone!!!
7227	*/
7228	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7229	{
7230	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7231
7232	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7233	return VINF_SUCCESS;
7234
7235	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7236	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7237
7238	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7239	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7240
7241	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7242	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7243
7244	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7245	return rc;
7246	}
7247
7248
7249	/**
7250	* Saves the host state in the VMCS host-state.
7251	*
7252	* @returns VBox status code.
7253	* @param pVM Pointer to the VM.
7254	* @param pVCpu Pointer to the VMCPU.
7255	*
7256	* @remarks No-long-jump zone!!!
7257	*/
7258	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7259	{
7260	AssertPtr(pVM);
7261	AssertPtr(pVCpu);
7262
7263	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7264
7265	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7266	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7267	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7268	return hmR0VmxSaveHostState(pVM, pVCpu);
7269	}
7270
7271
7272	/**
7273	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7274	* loaded from these fields on every successful VM-entry.
7275	*
7276	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7277	* Sets up the VM-entry controls.
7278	* Sets up the appropriate VMX non-root function to execute guest code based on
7279	* the guest CPU mode.
7280	*
7281	* @returns VBox status code.
7282	* @param pVM Pointer to the VM.
7283	* @param pVCpu Pointer to the VMCPU.
7284	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7285	* out-of-sync. Make sure to update the required fields
7286	* before using them.
7287	*
7288	* @remarks No-long-jump zone!!!
7289	*/
7290	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7291	{
7292	AssertPtr(pVM);
7293	AssertPtr(pVCpu);
7294	AssertPtr(pMixedCtx);
7295	HMVMX_ASSERT_PREEMPT_SAFE();
7296
7297	#ifdef LOG_ENABLED
7298	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7299	* probably not initialized yet? Anyway this will do for now.
7300	*
7301	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7302	* interface and disable ring-3 calls when thread-context hooks are not
7303	* available. */
7304	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7305	VMMR0LogFlushDisable(pVCpu);
7306	#endif
7307
7308	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7309
7310	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7311
7312	/* Determine real-on-v86 mode. */
7313	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7314	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7315	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7316	{
7317	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7318	}
7319
7320	/*
7321	* Load the guest-state into the VMCS.
7322	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7323	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7324	*/
7325	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7326	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7327
7328	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7329	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7330	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7331
7332	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7333	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7335
7336	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7338
7339	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7340	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7341
7342	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7343	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7344	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7345
7346	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7347	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7348
7349	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7350	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7351
7352	/*
7353	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7354	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7355	*/
7356	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7357	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7358
7359	/* Clear any unused and reserved bits. */
7360	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7361
7362	#ifdef LOG_ENABLED
7363	/* Only reenable log-flushing if the caller has it enabled. */
7364	if (!fCallerDisabledLogFlush)
7365	VMMR0LogFlushEnable(pVCpu);
7366	#endif
7367
7368	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7369	return rc;
7370	}
7371
7372
7373	/**
7374	* Loads the state shared between the host and guest into the VMCS.
7375	*
7376	* @param pVM Pointer to the VM.
7377	* @param pVCpu Pointer to the VMCPU.
7378	* @param pCtx Pointer to the guest-CPU context.
7379	*
7380	* @remarks No-long-jump zone!!!
7381	*/
7382	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7383	{
7384	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7385	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7386
7387	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7388	{
7389	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7390	AssertRC(rc);
7391	}
7392
7393	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7394	{
7395	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7396	AssertRC(rc);
7397
7398	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7399	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7400	{
7401	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7402	AssertRC(rc);
7403	}
7404	}
7405
7406	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7407	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7408	}
7409
7410
7411	/**
7412	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7413	*
7414	* @param pVM Pointer to the VM.
7415	* @param pVCpu Pointer to the VMCPU.
7416	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7417	* out-of-sync. Make sure to update the required fields
7418	* before using them.
7419	*/
7420	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7421	{
7422	HMVMX_ASSERT_PREEMPT_SAFE();
7423
7424	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7425	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7426	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7427	#endif
7428
7429	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7430	{
7431	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7432	AssertRC(rc);
7433	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7434	}
7435	else if (VMCPU_HMCF_VALUE(pVCpu))
7436	{
7437	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7438	AssertRC(rc);
7439	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7440	}
7441
7442	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7443	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7444	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7445	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7446
7447	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7448	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7449	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7450	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7451	#endif
7452	}
7453
7454
7455	/**
7456	* Does the preparations before executing guest code in VT-x.
7457	*
7458	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7459	* recompiler. We must be cautious what we do here regarding committing
7460	* guest-state information into the VMCS assuming we assuredly execute the
7461	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7462	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7463	* so that the recompiler can (and should) use them when it resumes guest
7464	* execution. Otherwise such operations must be done when we can no longer
7465	* exit to ring-3.
7466	*
7467	* @returns Strict VBox status code.
7468	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7469	* have been disabled.
7470	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7471	* double-fault into the guest.
7472	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7473	*
7474	* @param pVM Pointer to the VM.
7475	* @param pVCpu Pointer to the VMCPU.
7476	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7477	* out-of-sync. Make sure to update the required fields
7478	* before using them.
7479	* @param pVmxTransient Pointer to the VMX transient structure.
7480	*
7481	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7482	* interrupts will be disabled.
7483	*/
7484	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7485	{
7486	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7487
7488	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7489	PGMRZDynMapFlushAutoSet(pVCpu);
7490	#endif
7491
7492	/* Check force flag actions that might require us to go back to ring-3. */
7493	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7494	if (rc != VINF_SUCCESS)
7495	return rc;
7496
7497	#ifndef IEM_VERIFICATION_MODE_FULL
7498	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7499	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7500	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7501	{
7502	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7503	RTGCPHYS GCPhysApicBase;
7504	GCPhysApicBase = pMixedCtx->msrApicBase;
7505	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7506
7507	/* Unalias any existing mapping. */
7508	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7509	AssertRCReturn(rc, rc);
7510
7511	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7512	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7513	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7514	AssertRCReturn(rc, rc);
7515
7516	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7517	}
7518	#endif /* !IEM_VERIFICATION_MODE_FULL */
7519
7520	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7521	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7522
7523	/*
7524	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7525	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7526	*/
7527	if (TRPMHasTrap(pVCpu))
7528	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7529	else if (!pVCpu->hm.s.Event.fPending)
7530	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7531
7532	/*
7533	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7534	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7535	*/
7536	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7537	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7538	{
7539	Assert(rc == VINF_EM_RESET);
7540	return rc;
7541	}
7542
7543	/*
7544	* No longjmps to ring-3 from this point on!!!
7545	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7546	* This also disables flushing of the R0-logger instance (if any).
7547	*/
7548	VMMRZCallRing3Disable(pVCpu);
7549
7550	/*
7551	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7552	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7553	*
7554	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7555	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7556	*
7557	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7558	* executing guest code.
7559	*/
7560	pVmxTransient->uEflags = ASMIntDisableFlags();
7561	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7562	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7563	{
7564	hmR0VmxClearEventVmcs(pVCpu);
7565	ASMSetFlags(pVmxTransient->uEflags);
7566	VMMRZCallRing3Enable(pVCpu);
7567	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7568	return VINF_EM_RAW_TO_R3;
7569	}
7570	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7571	{
7572	hmR0VmxClearEventVmcs(pVCpu);
7573	ASMSetFlags(pVmxTransient->uEflags);
7574	VMMRZCallRing3Enable(pVCpu);
7575	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7576	return VINF_EM_RAW_INTERRUPT;
7577	}
7578
7579	/* We've injected any pending events. This is really the point of no return (to ring-3). */
7580	pVCpu->hm.s.Event.fPending = false;
7581
7582	return VINF_SUCCESS;
7583	}
7584
7585
7586	/**
7587	* Prepares to run guest code in VT-x and we've committed to doing so. This
7588	* means there is no backing out to ring-3 or anywhere else at this
7589	* point.
7590	*
7591	* @param pVM Pointer to the VM.
7592	* @param pVCpu Pointer to the VMCPU.
7593	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7594	* out-of-sync. Make sure to update the required fields
7595	* before using them.
7596	* @param pVmxTransient Pointer to the VMX transient structure.
7597	*
7598	* @remarks Called with preemption disabled.
7599	* @remarks No-long-jump zone!!!
7600	*/
7601	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7602	{
7603	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7604	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7605	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7606
7607	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7608	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7609
7610	/*
7611	* If we are injecting events to a real-on-v86 mode guest, we may have to update
7612	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7613	* Reload only the necessary state, the assertion will catch if other parts of the code
7614	* change.
7615	*/
7616	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7617	{
7618	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7619	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7620	}
7621
7622	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7623	if (!CPUMIsGuestFPUStateActive(pVCpu))
7624	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7625	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7626	#endif
7627
7628	/*
7629	* Load the host state bits as we may've been preempted (only happens when
7630	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7631	*/
7632	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7633	{
7634	/* This ASSUMES that pfnStartVM has been set up already. */
7635	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7636	AssertRC(rc);
7637	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7638	}
7639	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7640
7641	/*
7642	* Load the state shared between host and guest (FPU, debug).
7643	*/
7644	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7645	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7646	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7647
7648	/* Store status of the shared guest-host state at the time of VM-entry. */
7649	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7650	if (CPUMIsGuestInLongModeEx(pMixedCtx))
7651	{
7652	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7653	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7654	}
7655	else
7656	#endif
7657	{
7658	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7659	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7660	}
7661	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7662
7663	/*
7664	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7665	*/
7666	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7667	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7668
7669	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7670	RTCPUID idCurrentCpu = pCpu->idCpu;
7671	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7672	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
7673	{
7674	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7675	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7676	}
7677
7678	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7679	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7680	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7681	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7682
7683	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7684
7685	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7686	to start executing. */
7687
7688	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7689	/*
7690	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7691	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7692	*/
7693	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7694	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7695	{
7696	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7697	uint64_t u64HostTscAux = 0;
7698	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7699	AssertRC(rc2);
7700	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7701	}
7702	#endif
7703	}
7704
7705
7706	/**
7707	* Performs some essential restoration of state after running guest code in
7708	* VT-x.
7709	*
7710	* @param pVM Pointer to the VM.
7711	* @param pVCpu Pointer to the VMCPU.
7712	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7713	* out-of-sync. Make sure to update the required fields
7714	* before using them.
7715	* @param pVmxTransient Pointer to the VMX transient structure.
7716	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7717	*
7718	* @remarks Called with interrupts disabled, and returns with interrups enabled!
7719	*
7720	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7721	* unconditionally when it is safe to do so.
7722	*/
7723	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7724	{
7725	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7726
7727	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7728	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7729	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7730	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7731	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7732
7733	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7734	{
7735	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7736	/* Restore host's TSC_AUX. */
7737	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7738	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7739	#endif
7740	/** @todo Find a way to fix hardcoding a guestimate. */
7741	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7742	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7743	}
7744
7745	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7746	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7747	Assert(!(ASMGetFlags() & X86_EFL_IF));
7748	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7749
7750	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7751	if (CPUMIsGuestFPUStateActive(pVCpu))
7752	{
7753	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7754	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7755	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7756	}
7757	#endif
7758
7759	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7760	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7761	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7762	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7763
7764	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7765	uint32_t uExitReason;
7766	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7767	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7768	AssertRC(rc);
7769	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7770	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntInfo);
7771
7772	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7773	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7774	{
7775	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7776	pVmxTransient->fVMEntryFailed));
7777	return;
7778	}
7779
7780	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7781	{
7782	/* Update the guest interruptibility-state from the VMCS. */
7783	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7784	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7785	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7786	AssertRC(rc);
7787	#endif
7788	/*
7789	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7790	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7791	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7792	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7793	*/
7794	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7795	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7796	{
7797	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7798	AssertRC(rc);
7799	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
7800	}
7801	}
7802	}
7803
7804
7805
7806	/**
7807	* Runs the guest code using VT-x the normal way.
7808	*
7809	* @returns VBox status code.
7810	* @param pVM Pointer to the VM.
7811	* @param pVCpu Pointer to the VMCPU.
7812	* @param pCtx Pointer to the guest-CPU context.
7813	*
7814	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7815	* @remarks Called with preemption disabled.
7816	*/
7817	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7818	{
7819	VMXTRANSIENT VmxTransient;
7820	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7821	int rc = VERR_INTERNAL_ERROR_5;
7822	uint32_t cLoops = 0;
7823
7824	for (;; cLoops++)
7825	{
7826	Assert(!HMR0SuspendPending());
7827	HMVMX_ASSERT_CPU_SAFE();
7828
7829	/* Preparatory work for running guest code, this may force us to return
7830	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7831	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7832	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7833	if (rc != VINF_SUCCESS)
7834	break;
7835
7836	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7837	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7838	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7839
7840	/* Restore any residual host-state and save any bits shared between host
7841	and guest into the guest-CPU state. Re-enables interrupts! */
7842	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7843
7844	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7845	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7846	{
7847	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7848	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7849	return rc;
7850	}
7851
7852	/* Handle the VM-exit. */
7853	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7854	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7855	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7856	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7857	HMVMX_START_EXIT_DISPATCH_PROF();
7858	#ifdef HMVMX_USE_FUNCTION_TABLE
7859	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7860	#else
7861	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7862	#endif
7863	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7864	if (rc != VINF_SUCCESS)
7865	break;
7866	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7867	{
7868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7869	rc = VINF_EM_RAW_INTERRUPT;
7870	break;
7871	}
7872	}
7873
7874	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7875	return rc;
7876	}
7877
7878
7879	/**
7880	* Single steps guest code using VT-x.
7881	*
7882	* @returns VBox status code.
7883	* @param pVM Pointer to the VM.
7884	* @param pVCpu Pointer to the VMCPU.
7885	* @param pCtx Pointer to the guest-CPU context.
7886	*
7887	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7888	* @remarks Called with preemption disabled.
7889	*/
7890	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7891	{
7892	VMXTRANSIENT VmxTransient;
7893	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7894	int rc = VERR_INTERNAL_ERROR_5;
7895	uint32_t cLoops = 0;
7896	uint16_t uCsStart = pCtx->cs.Sel;
7897	uint64_t uRipStart = pCtx->rip;
7898
7899	for (;; cLoops++)
7900	{
7901	Assert(!HMR0SuspendPending());
7902	HMVMX_ASSERT_CPU_SAFE();
7903
7904	/* Preparatory work for running guest code, this may force us to return
7905	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7906	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7907	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7908	if (rc != VINF_SUCCESS)
7909	break;
7910
7911	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7912	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7913	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7914
7915	/* Restore any residual host-state and save any bits shared between host
7916	and guest into the guest-CPU state. Re-enables interrupts! */
7917	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7918
7919	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7920	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7921	{
7922	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7923	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7924	return rc;
7925	}
7926
7927	/* Handle the VM-exit. */
7928	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7930	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7931	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7932	HMVMX_START_EXIT_DISPATCH_PROF();
7933	#ifdef HMVMX_USE_FUNCTION_TABLE
7934	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7935	#else
7936	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7937	#endif
7938	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7939	if (rc != VINF_SUCCESS)
7940	break;
7941	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7942	{
7943	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7944	rc = VINF_EM_RAW_INTERRUPT;
7945	break;
7946	}
7947
7948	/*
7949	* Did the RIP change, if so, consider it a single step.
7950	* Otherwise, make sure one of the TFs gets set.
7951	*/
7952	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7953	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7954	AssertRCReturn(rc2, rc2);
7955	if ( pCtx->rip != uRipStart
7956	\|\| pCtx->cs.Sel != uCsStart)
7957	{
7958	rc = VINF_EM_DBG_STEPPED;
7959	break;
7960	}
7961	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7962	}
7963
7964	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7965	return rc;
7966	}
7967
7968
7969	/**
7970	* Runs the guest code using VT-x.
7971	*
7972	* @returns VBox status code.
7973	* @param pVM Pointer to the VM.
7974	* @param pVCpu Pointer to the VMCPU.
7975	* @param pCtx Pointer to the guest-CPU context.
7976	*
7977	* @remarks Called with preemption disabled.
7978	*/
7979	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7980	{
7981	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7982	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7983	HMVMX_ASSERT_PREEMPT_SAFE();
7984
7985	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7986
7987	int rc;
7988	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7989	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7990	else
7991	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7992
7993	if (rc == VERR_EM_INTERPRETER)
7994	rc = VINF_EM_RAW_EMULATE_INSTR;
7995	else if (rc == VINF_EM_RESET)
7996	rc = VINF_EM_TRIPLE_FAULT;
7997
7998	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7999	if (RT_FAILURE(rc2))
8000	{
8001	pVCpu->hm.s.u32HMError = rc;
8002	rc = rc2;
8003	}
8004	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8005	return rc;
8006	}
8007
8008
8009	#ifndef HMVMX_USE_FUNCTION_TABLE
8010	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8011	{
8012	int rc;
8013	switch (rcReason)
8014	{
8015	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8016	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8017	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8018	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8019	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8020	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8021	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8022	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8023	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8024	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8025	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8026	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8027	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8028	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8029	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8030	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8031	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8032	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8033	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8034	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8035	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8036	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8037	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8038	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8039	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8040	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8041	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8042	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8043	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8044	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8045	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8046	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8047	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8048
8049	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8050	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8051	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8052	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8053	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8054	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8055	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8056	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8057	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8058
8059	case VMX_EXIT_VMCALL:
8060	case VMX_EXIT_VMCLEAR:
8061	case VMX_EXIT_VMLAUNCH:
8062	case VMX_EXIT_VMPTRLD:
8063	case VMX_EXIT_VMPTRST:
8064	case VMX_EXIT_VMREAD:
8065	case VMX_EXIT_VMRESUME:
8066	case VMX_EXIT_VMWRITE:
8067	case VMX_EXIT_VMXOFF:
8068	case VMX_EXIT_VMXON:
8069	case VMX_EXIT_INVEPT:
8070	case VMX_EXIT_INVVPID:
8071	case VMX_EXIT_VMFUNC:
8072	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8073	break;
8074	default:
8075	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8076	break;
8077	}
8078	return rc;
8079	}
8080	#endif
8081
8082	#ifdef DEBUG
8083	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8084	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8085	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8086
8087	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8088	do \
8089	{ \
8090	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8091	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8092	} while (0)
8093
8094	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8095	do { \
8096	AssertPtr(pVCpu); \
8097	AssertPtr(pMixedCtx); \
8098	AssertPtr(pVmxTransient); \
8099	Assert(pVmxTransient->fVMEntryFailed == false); \
8100	Assert(ASMIntAreEnabled()); \
8101	HMVMX_ASSERT_PREEMPT_SAFE(); \
8102	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8103	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8104	HMVMX_ASSERT_PREEMPT_SAFE(); \
8105	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8106	HMVMX_ASSERT_PREEMPT_CPUID(); \
8107	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8108	} while (0)
8109
8110	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8111	do { \
8112	Log4Func(("\n")); \
8113	} while(0)
8114	#else /* Release builds */
8115	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8116	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8117	#endif
8118
8119
8120	/**
8121	* Advances the guest RIP after reading it from the VMCS.
8122	*
8123	* @returns VBox status code.
8124	* @param pVCpu Pointer to the VMCPU.
8125	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8126	* out-of-sync. Make sure to update the required fields
8127	* before using them.
8128	* @param pVmxTransient Pointer to the VMX transient structure.
8129	*
8130	* @remarks No-long-jump zone!!!
8131	*/
8132	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8133	{
8134	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8135	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8136	AssertRCReturn(rc, rc);
8137
8138	pMixedCtx->rip += pVmxTransient->cbInstr;
8139	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8140	return rc;
8141	}
8142
8143
8144	/**
8145	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8146	* and update error record fields accordingly.
8147	*
8148	* @return VMX_IGS_* return codes.
8149	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8150	* wrong with the guest state.
8151	*
8152	* @param pVM Pointer to the VM.
8153	* @param pVCpu Pointer to the VMCPU.
8154	* @param pCtx Pointer to the guest-CPU state.
8155	*/
8156	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8157	{
8158	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8159	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8160	uError = (err); \
8161	break; \
8162	} else do {} while (0)
8163	/* Duplicate of IEM_IS_CANONICAL(). */
8164	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8165
8166	int rc;
8167	uint32_t uError = VMX_IGS_ERROR;
8168	uint32_t u32Val;
8169	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8170
8171	do
8172	{
8173	/*
8174	* CR0.
8175	*/
8176	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8177	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8178	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8179	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8180	if (fUnrestrictedGuest)
8181	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8182
8183	uint32_t u32GuestCR0;
8184	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8185	AssertRCBreak(rc);
8186	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8187	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8188	if ( !fUnrestrictedGuest
8189	&& (u32GuestCR0 & X86_CR0_PG)
8190	&& !(u32GuestCR0 & X86_CR0_PE))
8191	{
8192	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8193	}
8194
8195	/*
8196	* CR4.
8197	*/
8198	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8199	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8200
8201	uint32_t u32GuestCR4;
8202	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8203	AssertRCBreak(rc);
8204	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8205	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8206
8207	/*
8208	* IA32_DEBUGCTL MSR.
8209	*/
8210	uint64_t u64Val;
8211	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8212	AssertRCBreak(rc);
8213	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8214	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8215	{
8216	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8217	}
8218	uint64_t u64DebugCtlMsr = u64Val;
8219
8220	#ifdef VBOX_STRICT
8221	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8222	AssertRCBreak(rc);
8223	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8224	#endif
8225	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8226
8227	/*
8228	* RIP and RFLAGS.
8229	*/
8230	uint32_t u32Eflags;
8231	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8232	if (HMVMX_IS_64BIT_HOST_MODE())
8233	{
8234	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8235	AssertRCBreak(rc);
8236	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8237	if ( !fLongModeGuest
8238	\|\| !pCtx->cs.Attr.n.u1Long)
8239	{
8240	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8241	}
8242	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8243	* must be identical if the "IA32e mode guest" VM-entry control is 1
8244	* and CS.L is 1. No check applies if the CPU supports 64
8245	* linear-address bits. */
8246
8247	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8248	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8249	AssertRCBreak(rc);
8250	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8251	VMX_IGS_RFLAGS_RESERVED);
8252	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8253	u32Eflags = u64Val;
8254	}
8255	else
8256	#endif
8257	{
8258	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8259	AssertRCBreak(rc);
8260	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8261	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8262	}
8263
8264	if ( fLongModeGuest
8265	\|\| ( fUnrestrictedGuest
8266	&& !(u32GuestCR0 & X86_CR0_PE)))
8267	{
8268	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8269	}
8270
8271	uint32_t u32EntryInfo;
8272	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8273	AssertRCBreak(rc);
8274	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8275	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8276	{
8277	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8278	}
8279
8280	/*
8281	* 64-bit checks.
8282	*/
8283	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8284	if (HMVMX_IS_64BIT_HOST_MODE())
8285	{
8286	if ( fLongModeGuest
8287	&& !fUnrestrictedGuest)
8288	{
8289	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8290	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8291	}
8292
8293	if ( !fLongModeGuest
8294	&& (u32GuestCR4 & X86_CR4_PCIDE))
8295	{
8296	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8297	}
8298
8299	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8300	* 51:32 beyond the processor's physical-address width are 0. */
8301
8302	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8303	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8304	{
8305	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8306	}
8307
8308	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8309	AssertRCBreak(rc);
8310	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8311
8312	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8313	AssertRCBreak(rc);
8314	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8315	}
8316	#endif
8317
8318	/*
8319	* PERF_GLOBAL MSR.
8320	*/
8321	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8322	{
8323	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8324	AssertRCBreak(rc);
8325	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8326	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8327	}
8328
8329	/*
8330	* PAT MSR.
8331	*/
8332	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8333	{
8334	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8335	AssertRCBreak(rc);
8336	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8337	for (unsigned i = 0; i < 8; i++)
8338	{
8339	uint8_t u8Val = (u64Val & 0x7);
8340	if ( u8Val != 0 /* UC */
8341	\|\| u8Val != 1 /* WC */
8342	\|\| u8Val != 4 /* WT */
8343	\|\| u8Val != 5 /* WP */
8344	\|\| u8Val != 6 /* WB */
8345	\|\| u8Val != 7 /* UC- */)
8346	{
8347	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8348	}
8349	u64Val >>= 3;
8350	}
8351	}
8352
8353	/*
8354	* EFER MSR.
8355	*/
8356	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8357	{
8358	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8359	AssertRCBreak(rc);
8360	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8361	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8362	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8363	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8364	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8365	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8366	}
8367
8368	/*
8369	* Segment registers.
8370	*/
8371	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8372	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8373	if (!(u32Eflags & X86_EFL_VM))
8374	{
8375	/* CS */
8376	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8377	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8378	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8379	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8380	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8381	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8382	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8383	/* CS cannot be loaded with NULL in protected mode. */
8384	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8385	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8386	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8387	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8388	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8389	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8390	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8391	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8392	else
8393	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8394
8395	/* SS */
8396	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8397	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8398	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8399	if ( !(pCtx->cr0 & X86_CR0_PE)
8400	\|\| pCtx->cs.Attr.n.u4Type == 3)
8401	{
8402	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8403	}
8404	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8405	{
8406	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8407	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8408	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8409	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8410	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8411	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8412	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8413	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8414	}
8415
8416	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8417	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8418	{
8419	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8420	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8421	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8422	\|\| pCtx->ds.Attr.n.u4Type > 11
8423	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8424	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8425	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8426	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8427	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8428	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8429	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8430	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8431	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8432	}
8433	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8434	{
8435	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8436	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8437	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8438	\|\| pCtx->es.Attr.n.u4Type > 11
8439	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8440	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8441	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8442	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8443	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8444	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8445	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8446	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8447	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8448	}
8449	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8450	{
8451	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8452	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8453	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8454	\|\| pCtx->fs.Attr.n.u4Type > 11
8455	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8456	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8457	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8458	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8459	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8460	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8461	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8462	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8463	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8464	}
8465	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8466	{
8467	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8468	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8469	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8470	\|\| pCtx->gs.Attr.n.u4Type > 11
8471	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8472	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8473	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8474	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8475	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8476	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8477	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8478	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8479	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8480	}
8481	/* 64-bit capable CPUs. */
8482	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8483	if (HMVMX_IS_64BIT_HOST_MODE())
8484	{
8485	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8486	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8487	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8488	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8489	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8490	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8491	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8492	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8493	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8494	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8495	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8496	}
8497	#endif
8498	}
8499	else
8500	{
8501	/* V86 mode checks. */
8502	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8503	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8504	{
8505	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8506	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8507	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8508	}
8509	else
8510	{
8511	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8512	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8513	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8514	}
8515
8516	/* CS */
8517	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8518	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8519	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8520	/* SS */
8521	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8522	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8523	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8524	/* DS */
8525	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8526	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8527	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8528	/* ES */
8529	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8530	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8531	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8532	/* FS */
8533	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8534	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8535	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8536	/* GS */
8537	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8538	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8539	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8540	/* 64-bit capable CPUs. */
8541	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8542	if (HMVMX_IS_64BIT_HOST_MODE())
8543	{
8544	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8545	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8546	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8547	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8548	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8549	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8550	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8551	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8552	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8553	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8554	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8555	}
8556	#endif
8557	}
8558
8559	/*
8560	* TR.
8561	*/
8562	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8563	/* 64-bit capable CPUs. */
8564	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8565	if (HMVMX_IS_64BIT_HOST_MODE())
8566	{
8567	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8568	}
8569	#endif
8570	if (fLongModeGuest)
8571	{
8572	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8573	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8574	}
8575	else
8576	{
8577	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8578	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8579	VMX_IGS_TR_ATTR_TYPE_INVALID);
8580	}
8581	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8582	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8583	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8584	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8585	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8586	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8587	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8588	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8589
8590	/*
8591	* GDTR and IDTR.
8592	*/
8593	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8594	if (HMVMX_IS_64BIT_HOST_MODE())
8595	{
8596	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8597	AssertRCBreak(rc);
8598	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8599
8600	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8601	AssertRCBreak(rc);
8602	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8603	}
8604	#endif
8605
8606	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8607	AssertRCBreak(rc);
8608	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8609
8610	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8611	AssertRCBreak(rc);
8612	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8613
8614	/*
8615	* Guest Non-Register State.
8616	*/
8617	/* Activity State. */
8618	uint32_t u32ActivityState;
8619	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8620	AssertRCBreak(rc);
8621	HMVMX_CHECK_BREAK( !u32ActivityState
8622	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8623	VMX_IGS_ACTIVITY_STATE_INVALID);
8624	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8625	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8626	uint32_t u32IntrState;
8627	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8628	AssertRCBreak(rc);
8629	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8630	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8631	{
8632	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8633	}
8634
8635	/** @todo Activity state and injecting interrupts. Left as a todo since we
8636	* currently don't use activity states but ACTIVE. */
8637
8638	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8639	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8640
8641	/* Guest interruptibility-state. */
8642	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8643	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8644	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8645	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8646	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8647	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8648	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8649	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8650	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8651	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8652	{
8653	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8654	{
8655	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8656	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8657	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8658	}
8659	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8660	{
8661	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8662	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8663	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8664	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8665	}
8666	}
8667	/** @todo Assumes the processor is not in SMM. */
8668	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8669	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8670	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8671	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8672	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8673	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8674	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8675	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8676	{
8677	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8678	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8679	}
8680
8681	/* Pending debug exceptions. */
8682	if (HMVMX_IS_64BIT_HOST_MODE())
8683	{
8684	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8685	AssertRCBreak(rc);
8686	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8687	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8688	u32Val = u64Val; /* For pending debug exceptions checks below. */
8689	}
8690	else
8691	{
8692	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8693	AssertRCBreak(rc);
8694	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8695	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8696	}
8697
8698	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8699	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8700	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8701	{
8702	if ( (u32Eflags & X86_EFL_TF)
8703	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8704	{
8705	/* Bit 14 is PendingDebug.BS. */
8706	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8707	}
8708	if ( !(u32Eflags & X86_EFL_TF)
8709	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8710	{
8711	/* Bit 14 is PendingDebug.BS. */
8712	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8713	}
8714	}
8715
8716	/* VMCS link pointer. */
8717	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8718	AssertRCBreak(rc);
8719	if (u64Val != UINT64_C(0xffffffffffffffff))
8720	{
8721	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8722	/** @todo Bits beyond the processor's physical-address width MBZ. */
8723	/** @todo 32-bit located in memory referenced by value of this field (as a
8724	* physical address) must contain the processor's VMCS revision ID. */
8725	/** @todo SMM checks. */
8726	}
8727
8728	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8729
8730	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8731	if (uError == VMX_IGS_ERROR)
8732	uError = VMX_IGS_REASON_NOT_FOUND;
8733	} while (0);
8734
8735	pVCpu->hm.s.u32HMError = uError;
8736	return uError;
8737
8738	#undef HMVMX_ERROR_BREAK
8739	#undef HMVMX_CHECK_BREAK
8740	#undef HMVMX_IS_CANONICAL
8741	}
8742
8743	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8744	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8745	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8746
8747	/** @name VM-exit handlers.
8748	* @{
8749	*/
8750
8751	/**
8752	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8753	*/
8754	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8755	{
8756	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8758	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8759	#if HC_ARCH_BITS == 64
8760	Assert(ASMIntAreEnabled());
8761	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8762	return VINF_SUCCESS;
8763	#endif
8764	return VINF_EM_RAW_INTERRUPT;
8765	}
8766
8767
8768	/**
8769	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8770	*/
8771	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8772	{
8773	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8774	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8775
8776	int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
8777	AssertRCReturn(rc, rc);
8778
8779	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
8780	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8781	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8782	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
8783
8784	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8785	{
8786	/*
8787	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8788	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8789	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8790	*
8791	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8792	*/
8793	VMXDispatchHostNmi();
8794	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8795	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8796	return VINF_SUCCESS;
8797	}
8798
8799	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8800	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8801	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8802	{
8803	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8804	return VINF_SUCCESS;
8805	}
8806	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8807	{
8808	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8809	return rc;
8810	}
8811
8812	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
8813	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
8814	switch (uIntType)
8815	{
8816	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8817	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8818	/* no break */
8819	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8820	{
8821	switch (uVector)
8822	{
8823	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8824	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8825	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8826	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8827	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8828	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8829	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8830	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8831	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8832	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8833	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8834	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8835	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8836	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8837	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8838	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8839	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8840	#endif
8841	default:
8842	{
8843	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8844	AssertRCReturn(rc, rc);
8845
8846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8847	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8848	{
8849	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8850	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8851	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8852
8853	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8854	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
8855	AssertRCReturn(rc, rc);
8856	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
8857	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
8858	0 /* GCPtrFaultAddress */);
8859	AssertRCReturn(rc, rc);
8860	}
8861	else
8862	{
8863	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8864	pVCpu->hm.s.u32HMError = uVector;
8865	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8866	}
8867	break;
8868	}
8869	}
8870	break;
8871	}
8872
8873	default:
8874	{
8875	pVCpu->hm.s.u32HMError = uExitIntInfo;
8876	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
8877	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
8878	break;
8879	}
8880	}
8881	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8882	return rc;
8883	}
8884
8885
8886	/**
8887	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8888	*/
8889	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8890	{
8891	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8892
8893	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8894	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8895	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8896	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8897	AssertRCReturn(rc, rc);
8898
8899	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8900	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8901	return VINF_SUCCESS;
8902	}
8903
8904
8905	/**
8906	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8907	*/
8908	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8909	{
8910	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8911	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8912	HMVMX_RETURN_UNEXPECTED_EXIT();
8913	}
8914
8915
8916	/**
8917	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8918	*/
8919	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8920	{
8921	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8922	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8923	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8924	}
8925
8926
8927	/**
8928	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8929	*/
8930	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8931	{
8932	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8934	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8935	}
8936
8937
8938	/**
8939	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8940	*/
8941	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8942	{
8943	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8944	PVM pVM = pVCpu->CTX_SUFF(pVM);
8945	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8946	if (RT_LIKELY(rc == VINF_SUCCESS))
8947	{
8948	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8949	Assert(pVmxTransient->cbInstr == 2);
8950	}
8951	else
8952	{
8953	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8954	rc = VERR_EM_INTERPRETER;
8955	}
8956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8957	return rc;
8958	}
8959
8960
8961	/**
8962	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8963	*/
8964	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8965	{
8966	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8967	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8968	AssertRCReturn(rc, rc);
8969
8970	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8971	return VINF_EM_RAW_EMULATE_INSTR;
8972
8973	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8974	HMVMX_RETURN_UNEXPECTED_EXIT();
8975	}
8976
8977
8978	/**
8979	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8980	*/
8981	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8982	{
8983	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8984	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8985	AssertRCReturn(rc, rc);
8986
8987	PVM pVM = pVCpu->CTX_SUFF(pVM);
8988	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8989	if (RT_LIKELY(rc == VINF_SUCCESS))
8990	{
8991	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8992	Assert(pVmxTransient->cbInstr == 2);
8993	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8994	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8995	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8996	}
8997	else
8998	{
8999	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9000	rc = VERR_EM_INTERPRETER;
9001	}
9002	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9003	return rc;
9004	}
9005
9006
9007	/**
9008	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9009	*/
9010	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9011	{
9012	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9013	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9014	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9015	AssertRCReturn(rc, rc);
9016
9017	PVM pVM = pVCpu->CTX_SUFF(pVM);
9018	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9019	if (RT_LIKELY(rc == VINF_SUCCESS))
9020	{
9021	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9022	Assert(pVmxTransient->cbInstr == 3);
9023	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9024	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9025	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9026	}
9027	else
9028	{
9029	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9030	rc = VERR_EM_INTERPRETER;
9031	}
9032	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9033	return rc;
9034	}
9035
9036
9037	/**
9038	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9039	*/
9040	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9041	{
9042	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9043	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9044	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9045	AssertRCReturn(rc, rc);
9046
9047	PVM pVM = pVCpu->CTX_SUFF(pVM);
9048	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9049	if (RT_LIKELY(rc == VINF_SUCCESS))
9050	{
9051	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9052	Assert(pVmxTransient->cbInstr == 2);
9053	}
9054	else
9055	{
9056	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9057	rc = VERR_EM_INTERPRETER;
9058	}
9059	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9060	return rc;
9061	}
9062
9063
9064	/**
9065	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9066	*/
9067	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9068	{
9069	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9070	PVM pVM = pVCpu->CTX_SUFF(pVM);
9071	Assert(!pVM->hm.s.fNestedPaging);
9072
9073	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9074	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9075	AssertRCReturn(rc, rc);
9076
9077	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9078	rc = VBOXSTRICTRC_VAL(rc2);
9079	if (RT_LIKELY(rc == VINF_SUCCESS))
9080	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9081	else
9082	{
9083	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9084	pVmxTransient->uExitQualification, rc));
9085	}
9086	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9087	return rc;
9088	}
9089
9090
9091	/**
9092	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9093	*/
9094	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9095	{
9096	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9097	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9098	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9099	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9100	AssertRCReturn(rc, rc);
9101
9102	PVM pVM = pVCpu->CTX_SUFF(pVM);
9103	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9104	if (RT_LIKELY(rc == VINF_SUCCESS))
9105	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9106	else
9107	{
9108	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9109	rc = VERR_EM_INTERPRETER;
9110	}
9111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9112	return rc;
9113	}
9114
9115
9116	/**
9117	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9118	*/
9119	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9120	{
9121	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9122	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9123	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9124	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9125	AssertRCReturn(rc, rc);
9126
9127	PVM pVM = pVCpu->CTX_SUFF(pVM);
9128	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9129	rc = VBOXSTRICTRC_VAL(rc2);
9130	if (RT_LIKELY( rc == VINF_SUCCESS
9131	\|\| rc == VINF_EM_HALT))
9132	{
9133	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9134	AssertRCReturn(rc3, rc3);
9135
9136	if ( rc == VINF_EM_HALT
9137	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9138	{
9139	rc = VINF_SUCCESS;
9140	}
9141	}
9142	else
9143	{
9144	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9145	rc = VERR_EM_INTERPRETER;
9146	}
9147	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9148	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9149	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9150	return rc;
9151	}
9152
9153
9154	/**
9155	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9156	*/
9157	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9158	{
9159	/*
9160	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9161	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9162	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9163	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9164	*/
9165	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9166	HMVMX_RETURN_UNEXPECTED_EXIT();
9167	}
9168
9169
9170	/**
9171	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9172	*/
9173	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9174	{
9175	/*
9176	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9177	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9178	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9179	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9180	*/
9181	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9182	HMVMX_RETURN_UNEXPECTED_EXIT();
9183	}
9184
9185
9186	/**
9187	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9188	*/
9189	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9190	{
9191	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9192	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9193	HMVMX_RETURN_UNEXPECTED_EXIT();
9194	}
9195
9196
9197	/**
9198	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9199	*/
9200	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9201	{
9202	/*
9203	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9204	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9205	* See Intel spec. 25.3 "Other Causes of VM-exits".
9206	*/
9207	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9208	HMVMX_RETURN_UNEXPECTED_EXIT();
9209	}
9210
9211
9212	/**
9213	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9214	* VM-exit.
9215	*/
9216	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9217	{
9218	/*
9219	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9220	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9221	*
9222	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9223	* See Intel spec. "23.8 Restrictions on VMX operation".
9224	*/
9225	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9226	return VINF_SUCCESS;
9227	}
9228
9229
9230	/**
9231	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9232	* VM-exit.
9233	*/
9234	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9235	{
9236	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9237	return VINF_EM_RESET;
9238	}
9239
9240
9241	/**
9242	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9243	*/
9244	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9245	{
9246	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9247	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9248	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9249	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9250	AssertRCReturn(rc, rc);
9251
9252	pMixedCtx->rip++;
9253	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9254	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9255	rc = VINF_SUCCESS;
9256	else
9257	rc = VINF_EM_HALT;
9258
9259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9260	return rc;
9261	}
9262
9263
9264	/**
9265	* VM-exit handler for instructions that result in a #UD exception delivered to
9266	* the guest.
9267	*/
9268	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9269	{
9270	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9271	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9272	return VINF_SUCCESS;
9273	}
9274
9275
9276	/**
9277	* VM-exit handler for expiry of the VMX preemption timer.
9278	*/
9279	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9280	{
9281	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9282
9283	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9284	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9285
9286	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9287	PVM pVM = pVCpu->CTX_SUFF(pVM);
9288	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9289	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9290	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9291	}
9292
9293
9294	/**
9295	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9296	*/
9297	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9298	{
9299	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9300
9301	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9302	/** @todo check if XSETBV is supported by the recompiler. */
9303	return VERR_EM_INTERPRETER;
9304	}
9305
9306
9307	/**
9308	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9309	*/
9310	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9311	{
9312	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9313
9314	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9315	/** @todo implement EMInterpretInvpcid() */
9316	return VERR_EM_INTERPRETER;
9317	}
9318
9319
9320	/**
9321	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9322	* Error VM-exit.
9323	*/
9324	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9325	{
9326	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9327	AssertRCReturn(rc, rc);
9328
9329	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9330	NOREF(uInvalidReason);
9331
9332	#ifdef VBOX_STRICT
9333	uint32_t uIntrState;
9334	HMVMXHCUINTREG uHCReg;
9335	uint64_t u64Val;
9336	uint32_t u32Val;
9337
9338	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9339	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9340	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9341	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9342	AssertRCReturn(rc, rc);
9343
9344	Log4(("uInvalidReason %u\n", uInvalidReason));
9345	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9346	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9347	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9348	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9349
9350	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9351	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9352	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9353	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9354	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9355	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9356	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9357	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9358	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9359	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9360	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9361	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9362	#endif
9363
9364	PVM pVM = pVCpu->CTX_SUFF(pVM);
9365	HMDumpRegs(pVM, pVCpu, pMixedCtx);
9366
9367	return VERR_VMX_INVALID_GUEST_STATE;
9368	}
9369
9370
9371	/**
9372	* VM-exit handler for VM-entry failure due to an MSR-load
9373	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9374	*/
9375	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9376	{
9377	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9378	HMVMX_RETURN_UNEXPECTED_EXIT();
9379	}
9380
9381
9382	/**
9383	* VM-exit handler for VM-entry failure due to a machine-check event
9384	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9385	*/
9386	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9387	{
9388	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9389	HMVMX_RETURN_UNEXPECTED_EXIT();
9390	}
9391
9392
9393	/**
9394	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9395	* theory.
9396	*/
9397	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9398	{
9399	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9400	return VERR_VMX_UNDEFINED_EXIT_CODE;
9401	}
9402
9403
9404	/**
9405	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9406	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9407	* Conditional VM-exit.
9408	*/
9409	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9410	{
9411	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9412
9413	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9414	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9415	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9416	return VERR_EM_INTERPRETER;
9417	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9418	HMVMX_RETURN_UNEXPECTED_EXIT();
9419	}
9420
9421
9422	/**
9423	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9424	*/
9425	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9426	{
9427	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9428
9429	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9430	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9431	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9432	return VERR_EM_INTERPRETER;
9433	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9434	HMVMX_RETURN_UNEXPECTED_EXIT();
9435	}
9436
9437
9438	/**
9439	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9440	*/
9441	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9442	{
9443	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9444
9445	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9446	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9447	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9448	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9449	AssertRCReturn(rc, rc);
9450	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9451
9452	PVM pVM = pVCpu->CTX_SUFF(pVM);
9453	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9454	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9455	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9456	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9457
9458	if (RT_LIKELY(rc == VINF_SUCCESS))
9459	{
9460	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9461	Assert(pVmxTransient->cbInstr == 2);
9462	}
9463	return rc;
9464	}
9465
9466
9467	/**
9468	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9469	*/
9470	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9471	{
9472	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9473	PVM pVM = pVCpu->CTX_SUFF(pVM);
9474	int rc = VINF_SUCCESS;
9475
9476	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9477	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9478	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9479	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9480	AssertRCReturn(rc, rc);
9481	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9482
9483	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9484	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9486
9487	if (RT_LIKELY(rc == VINF_SUCCESS))
9488	{
9489	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9490
9491	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9492	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9493	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9494	{
9495	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9496	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9497	EMInterpretWrmsr() changes it. */
9498	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9499	}
9500	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9501	{
9502	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9503	AssertRCReturn(rc, rc);
9504	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9505	}
9506	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9507	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9508
9509	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9510	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9511	{
9512	switch (pMixedCtx->ecx)
9513	{
9514	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9515	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9516	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9517	case MSR_K8_FS_BASE: /* no break */
9518	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9519	case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9520	}
9521	}
9522	#ifdef VBOX_STRICT
9523	else
9524	{
9525	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9526	switch (pMixedCtx->ecx)
9527	{
9528	case MSR_IA32_SYSENTER_CS:
9529	case MSR_IA32_SYSENTER_EIP:
9530	case MSR_IA32_SYSENTER_ESP:
9531	case MSR_K8_FS_BASE:
9532	case MSR_K8_GS_BASE:
9533	{
9534	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9535	HMVMX_RETURN_UNEXPECTED_EXIT();
9536	}
9537
9538	case MSR_K8_LSTAR:
9539	case MSR_K6_STAR:
9540	case MSR_K8_SF_MASK:
9541	case MSR_K8_TSC_AUX:
9542	case MSR_K8_KERNEL_GS_BASE:
9543	{
9544	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9545	pMixedCtx->ecx));
9546	HMVMX_RETURN_UNEXPECTED_EXIT();
9547	}
9548	}
9549	}
9550	#endif /* VBOX_STRICT */
9551	}
9552	return rc;
9553	}
9554
9555
9556	/**
9557	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9558	*/
9559	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9560	{
9561	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9562
9563	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9565	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9566	return VERR_EM_INTERPRETER;
9567	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9568	HMVMX_RETURN_UNEXPECTED_EXIT();
9569	}
9570
9571
9572	/**
9573	* VM-exit handler for when the TPR value is lowered below the specified
9574	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9575	*/
9576	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9577	{
9578	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9579	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9580
9581	/*
9582	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9583	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9584	* resume guest execution.
9585	*/
9586	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9587	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9588	return VINF_SUCCESS;
9589	}
9590
9591
9592	/**
9593	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9594	* VM-exit.
9595	*
9596	* @retval VINF_SUCCESS when guest execution can continue.
9597	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9598	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9599	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9600	* recompiler.
9601	*/
9602	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9603	{
9604	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9605	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9606	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9607	AssertRCReturn(rc, rc);
9608
9609	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9610	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9611	PVM pVM = pVCpu->CTX_SUFF(pVM);
9612	switch (uAccessType)
9613	{
9614	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9615	{
9616	#if 0
9617	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9618	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9619	#else
9620	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9621	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9622	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9623	#endif
9624	AssertRCReturn(rc, rc);
9625
9626	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9627	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9628	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9629	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9630
9631	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9632	{
9633	case 0: /* CR0 */
9634	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9635	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9636	break;
9637	case 2: /* C2 **/
9638	/* Nothing to do here, CR2 it's not part of the VMCS. */
9639	break;
9640	case 3: /* CR3 */
9641	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9642	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9643	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9644	break;
9645	case 4: /* CR4 */
9646	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9647	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9648	break;
9649	case 8: /* CR8 */
9650	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9651	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9652	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9653	break;
9654	default:
9655	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9656	break;
9657	}
9658
9659	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9660	break;
9661	}
9662
9663	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9664	{
9665	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9666	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9667	AssertRCReturn(rc, rc);
9668	Assert( !pVM->hm.s.fNestedPaging
9669	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9670	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9671
9672	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9673	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9674	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9675
9676	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9677	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9678	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9679	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9681	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9682	break;
9683	}
9684
9685	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9686	{
9687	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9688	AssertRCReturn(rc, rc);
9689	rc = EMInterpretCLTS(pVM, pVCpu);
9690	AssertRCReturn(rc, rc);
9691	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9692	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9693	Log4(("CRX CLTS write rc=%d\n", rc));
9694	break;
9695	}
9696
9697	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9698	{
9699	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9700	AssertRCReturn(rc, rc);
9701	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9702	if (RT_LIKELY(rc == VINF_SUCCESS))
9703	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9705	Log4(("CRX LMSW write rc=%d\n", rc));
9706	break;
9707	}
9708
9709	default:
9710	{
9711	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9712	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9713	}
9714	}
9715
9716	/* Validate possible error codes. */
9717	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9718	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9719	if (RT_SUCCESS(rc))
9720	{
9721	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9722	AssertRCReturn(rc2, rc2);
9723	}
9724
9725	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9726	return rc;
9727	}
9728
9729
9730	/**
9731	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9732	* VM-exit.
9733	*/
9734	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9735	{
9736	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9737	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9738
9739	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9740	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9741	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9742	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9743	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9744	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9745	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9746	AssertRCReturn(rc2, rc2);
9747
9748	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9749	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9750	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9751	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9752	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9753	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9754	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9755
9756	/* I/O operation lookup arrays. */
9757	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9758	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9759
9760	VBOXSTRICTRC rcStrict;
9761	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9762	const uint32_t cbInstr = pVmxTransient->cbInstr;
9763	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9764	PVM pVM = pVCpu->CTX_SUFF(pVM);
9765	if (fIOString)
9766	{
9767	/*
9768	* INS/OUTS - I/O String instruction.
9769	*
9770	* Use instruction-information if available, otherwise fall back on
9771	* interpreting the instruction.
9772	*/
9773	#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9774	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9775	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9776	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9777	{
9778	rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9779	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9780	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9781	AssertRCReturn(rc2, rc2);
9782	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9783	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9784	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9785	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9786	if (fIOWrite)
9787	{
9788	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9789	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9790	//if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9791	// hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9792	// pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9793	}
9794	else
9795	{
9796	AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9797	("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9798	VERR_HMVMX_IPE_4);
9799	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9800	//if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9801	// hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9802	}
9803	}
9804	else
9805	{
9806	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9807	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9808	AssertRCReturn(rc2, rc2);
9809	rcStrict = IEMExecOne(pVCpu);
9810	}
9811	/** @todo IEM needs to be setting these flags somehow. */
9812	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9813	fUpdateRipAlready = true;
9814	#else
9815	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9816	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9817	if (RT_SUCCESS(rcStrict))
9818	{
9819	if (fIOWrite)
9820	{
9821	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9822	(DISCPUMODE)pDis->uAddrMode, cbValue);
9823	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9824	}
9825	else
9826	{
9827	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9828	(DISCPUMODE)pDis->uAddrMode, cbValue);
9829	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9830	}
9831	}
9832	else
9833	{
9834	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9835	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9836	}
9837	#endif
9838	}
9839	else
9840	{
9841	/*
9842	* IN/OUT - I/O instruction.
9843	*/
9844	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9845	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9846	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9847	if (fIOWrite)
9848	{
9849	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9850	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9851	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9852	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9853	}
9854	else
9855	{
9856	uint32_t u32Result = 0;
9857	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9858	if (IOM_SUCCESS(rcStrict))
9859	{
9860	/* Save result of I/O IN instr. in AL/AX/EAX. */
9861	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9862	}
9863	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9864	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9865	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9866	}
9867	}
9868
9869	if (IOM_SUCCESS(rcStrict))
9870	{
9871	if (!fUpdateRipAlready)
9872	{
9873	pMixedCtx->rip += cbInstr;
9874	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9875	}
9876
9877	/*
9878	* If any I/O breakpoints are armed, we need to check if one triggered
9879	* and take appropriate action.
9880	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9881	*/
9882	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9883	AssertRCReturn(rc2, rc2);
9884
9885	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9886	* execution engines about whether hyper BPs and such are pending. */
9887	uint32_t const uDr7 = pMixedCtx->dr[7];
9888	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9889	&& X86_DR7_ANY_RW_IO(uDr7)
9890	&& (pMixedCtx->cr4 & X86_CR4_DE))
9891	\|\| DBGFBpIsHwIoArmed(pVM)))
9892	{
9893	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9894
9895	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
9896	VMMRZCallRing3Disable(pVCpu);
9897	HM_DISABLE_PREEMPT_IF_NEEDED();
9898
9899	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9900
9901	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9902	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9903	{
9904	/* Raise #DB. */
9905	if (fIsGuestDbgActive)
9906	ASMSetDR6(pMixedCtx->dr[6]);
9907	if (pMixedCtx->dr[7] != uDr7)
9908	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9909
9910	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9911	}
9912	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9913	else if ( rcStrict2 != VINF_SUCCESS
9914	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9915	rcStrict = rcStrict2;
9916
9917	HM_RESTORE_PREEMPT_IF_NEEDED();
9918	VMMRZCallRing3Enable(pVCpu);
9919	}
9920	}
9921
9922	#ifdef DEBUG
9923	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9924	Assert(!fIOWrite);
9925	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9926	Assert(fIOWrite);
9927	else
9928	{
9929	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9930	* statuses, that the VMM device and some others may return. See
9931	* IOM_SUCCESS() for guidance. */
9932	AssertMsg( RT_FAILURE(rcStrict)
9933	\|\| rcStrict == VINF_SUCCESS
9934	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9935	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9936	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9937	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9938	}
9939	#endif
9940
9941	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9942	return VBOXSTRICTRC_TODO(rcStrict);
9943	}
9944
9945
9946	/**
9947	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9948	* VM-exit.
9949	*/
9950	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9951	{
9952	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9953
9954	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9955	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9956	AssertRCReturn(rc, rc);
9957	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9958	{
9959	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9960	AssertRCReturn(rc, rc);
9961	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9962	{
9963	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9964
9965	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9966	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9967	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9968	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9969	{
9970	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9971	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9972
9973	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9974	Assert(!pVCpu->hm.s.Event.fPending);
9975	pVCpu->hm.s.Event.fPending = true;
9976	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
9977	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9978	AssertRCReturn(rc, rc);
9979	if (fErrorCodeValid)
9980	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9981	else
9982	pVCpu->hm.s.Event.u32ErrCode = 0;
9983	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9984	&& uVector == X86_XCPT_PF)
9985	{
9986	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9987	}
9988
9989	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9990	}
9991	}
9992	}
9993
9994	/** @todo Emulate task switch someday, currently just going back to ring-3 for
9995	* emulation. */
9996	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9997	return VERR_EM_INTERPRETER;
9998	}
9999
10000
10001	/**
10002	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10003	*/
10004	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10005	{
10006	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10007	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10008	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10009	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10010	AssertRCReturn(rc, rc);
10011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10012	return VINF_EM_DBG_STEPPED;
10013	}
10014
10015
10016	/**
10017	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10018	*/
10019	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10020	{
10021	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10022
10023	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10024	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10025	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10026	return VINF_SUCCESS;
10027	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10028	return rc;
10029
10030	#if 0
10031	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10032	* just sync the whole thing. */
10033	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10034	#else
10035	/* Aggressive state sync. for now. */
10036	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10037	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10038	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10039	#endif
10040	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10041	AssertRCReturn(rc, rc);
10042
10043	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10044	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10045	switch (uAccessType)
10046	{
10047	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10048	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10049	{
10050	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10051	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10052	{
10053	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10054	}
10055
10056	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10057	GCPhys &= PAGE_BASE_GC_MASK;
10058	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10059	PVM pVM = pVCpu->CTX_SUFF(pVM);
10060	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10061	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10062
10063	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10064	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10065	CPUMCTX2CORE(pMixedCtx), GCPhys);
10066	rc = VBOXSTRICTRC_VAL(rc2);
10067	Log4(("ApicAccess rc=%d\n", rc));
10068	if ( rc == VINF_SUCCESS
10069	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10070	\|\| rc == VERR_PAGE_NOT_PRESENT)
10071	{
10072	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10073	\| HM_CHANGED_GUEST_RSP
10074	\| HM_CHANGED_GUEST_RFLAGS
10075	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10076	rc = VINF_SUCCESS;
10077	}
10078	break;
10079	}
10080
10081	default:
10082	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10083	rc = VINF_EM_RAW_EMULATE_INSTR;
10084	break;
10085	}
10086
10087	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10088	return rc;
10089	}
10090
10091
10092	/**
10093	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10094	* VM-exit.
10095	*/
10096	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10097	{
10098	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10099
10100	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10101	if (pVmxTransient->fWasGuestDebugStateActive)
10102	{
10103	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10104	HMVMX_RETURN_UNEXPECTED_EXIT();
10105	}
10106
10107	int rc = VERR_INTERNAL_ERROR_5;
10108	if ( !DBGFIsStepping(pVCpu)
10109	&& !pVCpu->hm.s.fSingleInstruction
10110	&& !pVmxTransient->fWasHyperDebugStateActive)
10111	{
10112	/* Don't intercept MOV DRx and #DB any more. */
10113	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10114	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10115	AssertRCReturn(rc, rc);
10116
10117	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10118	{
10119	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10120	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10121	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10122	AssertRCReturn(rc, rc);
10123	#endif
10124	}
10125
10126	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10127	VMMRZCallRing3Disable(pVCpu);
10128	HM_DISABLE_PREEMPT_IF_NEEDED();
10129
10130	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10131	PVM pVM = pVCpu->CTX_SUFF(pVM);
10132	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10133	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10134
10135	HM_RESTORE_PREEMPT_IF_NEEDED();
10136	VMMRZCallRing3Enable(pVCpu);
10137
10138	#ifdef VBOX_WITH_STATISTICS
10139	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10140	AssertRCReturn(rc, rc);
10141	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10142	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10143	else
10144	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10145	#endif
10146	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10147	return VINF_SUCCESS;
10148	}
10149
10150	/*
10151	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10152	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10153	*/
10154	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10155	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10156	AssertRCReturn(rc, rc);
10157	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10158
10159	PVM pVM = pVCpu->CTX_SUFF(pVM);
10160	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10161	{
10162	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10163	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10164	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10165	if (RT_SUCCESS(rc))
10166	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10167	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10168	}
10169	else
10170	{
10171	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10172	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10173	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10175	}
10176
10177	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10178	if (RT_SUCCESS(rc))
10179	{
10180	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10181	AssertRCReturn(rc2, rc2);
10182	}
10183	return rc;
10184	}
10185
10186
10187	/**
10188	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10189	* Conditional VM-exit.
10190	*/
10191	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10192	{
10193	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10194	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10195
10196	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10197	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10198	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10199	return VINF_SUCCESS;
10200	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10201	return rc;
10202
10203	RTGCPHYS GCPhys = 0;
10204	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10205
10206	#if 0
10207	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10208	#else
10209	/* Aggressive state sync. for now. */
10210	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10211	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10212	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10213	#endif
10214	AssertRCReturn(rc, rc);
10215
10216	/*
10217	* If we succeed, resume guest execution.
10218	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10219	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10220	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10221	* weird case. See @bugref{6043}.
10222	*/
10223	PVM pVM = pVCpu->CTX_SUFF(pVM);
10224	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10225	rc = VBOXSTRICTRC_VAL(rc2);
10226	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10227	if ( rc == VINF_SUCCESS
10228	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10229	\|\| rc == VERR_PAGE_NOT_PRESENT)
10230	{
10231	/* Successfully handled MMIO operation. */
10232	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10233	\| HM_CHANGED_GUEST_RSP
10234	\| HM_CHANGED_GUEST_RFLAGS
10235	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10236	rc = VINF_SUCCESS;
10237	}
10238	return rc;
10239	}
10240
10241
10242	/**
10243	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10244	* VM-exit.
10245	*/
10246	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10247	{
10248	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10249	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10250
10251	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10252	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10253	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10254	return VINF_SUCCESS;
10255	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10256	return rc;
10257
10258	RTGCPHYS GCPhys = 0;
10259	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10260	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10261	#if 0
10262	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10263	#else
10264	/* Aggressive state sync. for now. */
10265	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10266	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10267	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10268	#endif
10269	AssertRCReturn(rc, rc);
10270
10271	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10272	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10273
10274	RTGCUINT uErrorCode = 0;
10275	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10276	uErrorCode \|= X86_TRAP_PF_ID;
10277	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10278	uErrorCode \|= X86_TRAP_PF_RW;
10279	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10280	uErrorCode \|= X86_TRAP_PF_P;
10281
10282	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10283
10284	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10285	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10286
10287	/* Handle the pagefault trap for the nested shadow table. */
10288	PVM pVM = pVCpu->CTX_SUFF(pVM);
10289	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10290	TRPMResetTrap(pVCpu);
10291
10292	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10293	if ( rc == VINF_SUCCESS
10294	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10295	\|\| rc == VERR_PAGE_NOT_PRESENT)
10296	{
10297	/* Successfully synced our nested page tables. */
10298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10299	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10300	\| HM_CHANGED_GUEST_RSP
10301	\| HM_CHANGED_GUEST_RFLAGS);
10302	return VINF_SUCCESS;
10303	}
10304
10305	Log4(("EPT return to ring-3 rc=%d\n"));
10306	return rc;
10307	}
10308
10309	/** @} */
10310
10311	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10312	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10313	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10314
10315	/** @name VM-exit exception handlers.
10316	* @{
10317	*/
10318
10319	/**
10320	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10321	*/
10322	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10323	{
10324	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10325	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10326
10327	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10328	AssertRCReturn(rc, rc);
10329
10330	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10331	{
10332	/* Old-style FPU error reporting needs some extra work. */
10333	/** @todo don't fall back to the recompiler, but do it manually. */
10334	return VERR_EM_INTERPRETER;
10335	}
10336
10337	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10338	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10339	return rc;
10340	}
10341
10342
10343	/**
10344	* VM-exit exception handler for #BP (Breakpoint exception).
10345	*/
10346	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10347	{
10348	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10350
10351	/** @todo Try optimize this by not saving the entire guest state unless
10352	* really needed. */
10353	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10354	AssertRCReturn(rc, rc);
10355
10356	PVM pVM = pVCpu->CTX_SUFF(pVM);
10357	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10358	if (rc == VINF_EM_RAW_GUEST_TRAP)
10359	{
10360	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10361	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10362	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10363	AssertRCReturn(rc, rc);
10364
10365	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10366	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10367	}
10368
10369	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10370	return rc;
10371	}
10372
10373
10374	/**
10375	* VM-exit exception handler for #DB (Debug exception).
10376	*/
10377	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10378	{
10379	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10381	Log6(("XcptDB\n"));
10382
10383	/*
10384	* Get the DR6-like values from the exit qualification and pass it to DBGF
10385	* for processing.
10386	*/
10387	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10388	AssertRCReturn(rc, rc);
10389
10390	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10391	uint64_t uDR6 = X86_DR6_INIT_VAL;
10392	uDR6 \|= ( pVmxTransient->uExitQualification
10393	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10394
10395	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10396	if (rc == VINF_EM_RAW_GUEST_TRAP)
10397	{
10398	/*
10399	* The exception was for the guest. Update DR6, DR7.GD and
10400	* IA32_DEBUGCTL.LBR before forwarding it.
10401	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10402	*/
10403	VMMRZCallRing3Disable(pVCpu);
10404	HM_DISABLE_PREEMPT_IF_NEEDED();
10405
10406	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10407	pMixedCtx->dr[6] \|= uDR6;
10408	if (CPUMIsGuestDebugStateActive(pVCpu))
10409	ASMSetDR6(pMixedCtx->dr[6]);
10410
10411	HM_RESTORE_PREEMPT_IF_NEEDED();
10412	VMMRZCallRing3Enable(pVCpu);
10413
10414	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10415	AssertRCReturn(rc, rc);
10416
10417	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10418	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10419
10420	/* Paranoia. */
10421	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10422	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10423
10424	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10425	AssertRCReturn(rc, rc);
10426
10427	/*
10428	* Raise #DB in the guest.
10429	*/
10430	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10431	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10432	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10433	AssertRCReturn(rc, rc);
10434	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10435	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10436	return VINF_SUCCESS;
10437	}
10438
10439	/*
10440	* Not a guest trap, must be a hypervisor related debug event then.
10441	* Update DR6 in case someone is interested in it.
10442	*/
10443	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10444	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10445	CPUMSetHyperDR6(pVCpu, uDR6);
10446
10447	return rc;
10448	}
10449
10450
10451	/**
10452	* VM-exit exception handler for #NM (Device-not-available exception: floating
10453	* point exception).
10454	*/
10455	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10456	{
10457	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10458
10459	/* We require CR0 and EFER. EFER is always up-to-date. */
10460	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10461	AssertRCReturn(rc, rc);
10462
10463	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10464	VMMRZCallRing3Disable(pVCpu);
10465	HM_DISABLE_PREEMPT_IF_NEEDED();
10466
10467	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10468	if (pVmxTransient->fWasGuestFPUStateActive)
10469	{
10470	rc = VINF_EM_RAW_GUEST_TRAP;
10471	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10472	}
10473	else
10474	{
10475	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10476	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10477	#endif
10478	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10479	rc = CPUMR0LoadGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10480	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10481	}
10482
10483	HM_RESTORE_PREEMPT_IF_NEEDED();
10484	VMMRZCallRing3Enable(pVCpu);
10485
10486	if (rc == VINF_SUCCESS)
10487	{
10488	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10489	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10490	}
10491	else
10492	{
10493	/* Forward #NM to the guest. */
10494	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10495	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10496	AssertRCReturn(rc, rc);
10497	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10498	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
10499	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10500	}
10501
10502	return VINF_SUCCESS;
10503	}
10504
10505
10506	/**
10507	* VM-exit exception handler for #GP (General-protection exception).
10508	*
10509	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10510	*/
10511	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10512	{
10513	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10514	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10515
10516	int rc = VERR_INTERNAL_ERROR_5;
10517	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10518	{
10519	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10520	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10521	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10522	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10523	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10524	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10525	AssertRCReturn(rc, rc);
10526	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10527	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10528	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10529	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10530	return rc;
10531	#else
10532	/* We don't intercept #GP. */
10533	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10534	return VERR_VMX_UNEXPECTED_EXCEPTION;
10535	#endif
10536	}
10537
10538	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10539	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10540
10541	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10542	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10543	AssertRCReturn(rc, rc);
10544
10545	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10546	uint32_t cbOp = 0;
10547	PVM pVM = pVCpu->CTX_SUFF(pVM);
10548	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10549	if (RT_SUCCESS(rc))
10550	{
10551	rc = VINF_SUCCESS;
10552	Assert(cbOp == pDis->cbInstr);
10553	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10554	switch (pDis->pCurInstr->uOpcode)
10555	{
10556	case OP_CLI:
10557	{
10558	pMixedCtx->eflags.Bits.u1IF = 0;
10559	pMixedCtx->rip += pDis->cbInstr;
10560	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10561	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10562	break;
10563	}
10564
10565	case OP_STI:
10566	{
10567	pMixedCtx->eflags.Bits.u1IF = 1;
10568	pMixedCtx->rip += pDis->cbInstr;
10569	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10570	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10571	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10572	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10573	break;
10574	}
10575
10576	case OP_HLT:
10577	{
10578	rc = VINF_EM_HALT;
10579	pMixedCtx->rip += pDis->cbInstr;
10580	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10581	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10582	break;
10583	}
10584
10585	case OP_POPF:
10586	{
10587	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10588	uint32_t cbParm = 0;
10589	uint32_t uMask = 0;
10590	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10591	{
10592	cbParm = 4;
10593	uMask = 0xffffffff;
10594	}
10595	else
10596	{
10597	cbParm = 2;
10598	uMask = 0xffff;
10599	}
10600
10601	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10602	RTGCPTR GCPtrStack = 0;
10603	X86EFLAGS Eflags;
10604	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10605	&GCPtrStack);
10606	if (RT_SUCCESS(rc))
10607	{
10608	Assert(sizeof(Eflags.u32) >= cbParm);
10609	Eflags.u32 = 0;
10610	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10611	}
10612	if (RT_FAILURE(rc))
10613	{
10614	rc = VERR_EM_INTERPRETER;
10615	break;
10616	}
10617	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10618	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10619	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10620	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10621	pMixedCtx->esp += cbParm;
10622	pMixedCtx->esp &= uMask;
10623	pMixedCtx->rip += pDis->cbInstr;
10624
10625	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10626	\| HM_CHANGED_GUEST_RSP
10627	\| HM_CHANGED_GUEST_RFLAGS);
10628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10629	break;
10630	}
10631
10632	case OP_PUSHF:
10633	{
10634	uint32_t cbParm = 0;
10635	uint32_t uMask = 0;
10636	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10637	{
10638	cbParm = 4;
10639	uMask = 0xffffffff;
10640	}
10641	else
10642	{
10643	cbParm = 2;
10644	uMask = 0xffff;
10645	}
10646
10647	/* Get the stack pointer & push the contents of eflags onto the stack. */
10648	RTGCPTR GCPtrStack = 0;
10649	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10650	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10651	if (RT_FAILURE(rc))
10652	{
10653	rc = VERR_EM_INTERPRETER;
10654	break;
10655	}
10656	X86EFLAGS Eflags = pMixedCtx->eflags;
10657	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10658	Eflags.Bits.u1RF = 0;
10659	Eflags.Bits.u1VM = 0;
10660
10661	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10662	if (RT_FAILURE(rc))
10663	{
10664	rc = VERR_EM_INTERPRETER;
10665	break;
10666	}
10667	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10668	pMixedCtx->esp -= cbParm;
10669	pMixedCtx->esp &= uMask;
10670	pMixedCtx->rip += pDis->cbInstr;
10671	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
10672	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10673	break;
10674	}
10675
10676	case OP_IRET:
10677	{
10678	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10679	* instruction reference. */
10680	RTGCPTR GCPtrStack = 0;
10681	uint32_t uMask = 0xffff;
10682	uint16_t aIretFrame[3];
10683	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10684	{
10685	rc = VERR_EM_INTERPRETER;
10686	break;
10687	}
10688	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10689	&GCPtrStack);
10690	if (RT_SUCCESS(rc))
10691	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10692	if (RT_FAILURE(rc))
10693	{
10694	rc = VERR_EM_INTERPRETER;
10695	break;
10696	}
10697	pMixedCtx->eip = 0;
10698	pMixedCtx->ip = aIretFrame[0];
10699	pMixedCtx->cs.Sel = aIretFrame[1];
10700	pMixedCtx->cs.ValidSel = aIretFrame[1];
10701	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10702	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10703	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10704	pMixedCtx->sp += sizeof(aIretFrame);
10705	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10706	\| HM_CHANGED_GUEST_SEGMENT_REGS
10707	\| HM_CHANGED_GUEST_RSP
10708	\| HM_CHANGED_GUEST_RFLAGS);
10709	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10710	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10711	break;
10712	}
10713
10714	case OP_INT:
10715	{
10716	uint16_t uVector = pDis->Param1.uValue & 0xff;
10717	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10719	break;
10720	}
10721
10722	case OP_INTO:
10723	{
10724	if (pMixedCtx->eflags.Bits.u1OF)
10725	{
10726	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10727	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10728	}
10729	break;
10730	}
10731
10732	default:
10733	{
10734	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10735	EMCODETYPE_SUPERVISOR);
10736	rc = VBOXSTRICTRC_VAL(rc2);
10737	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10738	Log4(("#GP rc=%Rrc\n", rc));
10739	break;
10740	}
10741	}
10742	}
10743	else
10744	rc = VERR_EM_INTERPRETER;
10745
10746	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10747	("#GP Unexpected rc=%Rrc\n", rc));
10748	return rc;
10749	}
10750
10751
10752	/**
10753	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10754	* the exception reported in the VMX transient structure back into the VM.
10755	*
10756	* @remarks Requires uExitIntInfo in the VMX transient structure to be
10757	* up-to-date.
10758	*/
10759	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10760	{
10761	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10762
10763	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10764	hmR0VmxCheckExitDueToEventDelivery(). */
10765	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10766	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10767	AssertRCReturn(rc, rc);
10768	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10769
10770	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10771	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10772	return VINF_SUCCESS;
10773	}
10774
10775
10776	/**
10777	* VM-exit exception handler for #PF (Page-fault exception).
10778	*/
10779	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10780	{
10781	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10782	PVM pVM = pVCpu->CTX_SUFF(pVM);
10783	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10784	rc \|= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10785	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10786	AssertRCReturn(rc, rc);
10787
10788	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10789	if (pVM->hm.s.fNestedPaging)
10790	{
10791	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10792	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10793	{
10794	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10795	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10796	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
10797	}
10798	else
10799	{
10800	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10801	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10802	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10803	}
10804	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10805	return rc;
10806	}
10807	#else
10808	Assert(!pVM->hm.s.fNestedPaging);
10809	#endif
10810
10811	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10812	AssertRCReturn(rc, rc);
10813
10814	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10815	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
10816
10817	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
10818	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
10819	(RTGCPTR)pVmxTransient->uExitQualification);
10820
10821	Log4(("#PF: rc=%Rrc\n", rc));
10822	if (rc == VINF_SUCCESS)
10823	{
10824	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10825	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10826	* memory? We don't update the whole state here... */
10827	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10828	\| HM_CHANGED_GUEST_RSP
10829	\| HM_CHANGED_GUEST_RFLAGS
10830	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10831	TRPMResetTrap(pVCpu);
10832	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10833	return rc;
10834	}
10835	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10836	{
10837	if (!pVmxTransient->fVectoringPF)
10838	{
10839	/* It's a guest page fault and needs to be reflected to the guest. */
10840	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10841	TRPMResetTrap(pVCpu);
10842	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10843	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10844	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10845	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10846	}
10847	else
10848	{
10849	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10850	TRPMResetTrap(pVCpu);
10851	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10852	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10853	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10854	}
10855
10856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10857	return VINF_SUCCESS;
10858	}
10859
10860	TRPMResetTrap(pVCpu);
10861	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10862	return rc;
10863	}
10864
10865	/** @} */
10866

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49000

Download in other formats: