HMVMXR0.cpp@ 64856

Last change on this file since 64856 was 64856, checked in by vboxsync, 8 years ago
VMM/HMVMXR0: Comment to clarify VM-exit history updates for 3 edge cases.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 585.2 KB

Line
1	/* $Id: HMVMXR0.cpp 64856 2016-12-13 17:28:40Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2016 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#ifdef DEBUG_ramshankar
44	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49	# define HMVMX_ALWAYS_TRAP_PF
50	# define HMVMX_ALWAYS_SWAP_FPU_STATE
51	# define HMVMX_ALWAYS_FLUSH_TLB
52	# define HMVMX_ALWAYS_SWAP_EFER
53	#endif
54
55
56	/*********************************************************************************************************************************
57	* Defined Constants And Macros *
58	*********************************************************************************************************************************/
59	/** Use the function table. */
60	#define HMVMX_USE_FUNCTION_TABLE
61
62	/** Determine which tagged-TLB flush handler to use. */
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68	/** @name Updated-guest-state flags.
69	* @{ */
70	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91	\| HMVMX_UPDATED_GUEST_RSP \
92	\| HMVMX_UPDATED_GUEST_RFLAGS \
93	\| HMVMX_UPDATED_GUEST_CR0 \
94	\| HMVMX_UPDATED_GUEST_CR3 \
95	\| HMVMX_UPDATED_GUEST_CR4 \
96	\| HMVMX_UPDATED_GUEST_GDTR \
97	\| HMVMX_UPDATED_GUEST_IDTR \
98	\| HMVMX_UPDATED_GUEST_LDTR \
99	\| HMVMX_UPDATED_GUEST_TR \
100	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101	\| HMVMX_UPDATED_GUEST_DEBUG \
102	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
107	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108	\| HMVMX_UPDATED_GUEST_INTR_STATE \
109	\| HMVMX_UPDATED_GUEST_APIC_STATE)
110	/** @} */
111
112	/** @name
113	* Flags to skip redundant reads of some common VMCS fields that are not part of
114	* the guest-CPU state but are in the transient structure.
115	*/
116	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123	/** @} */
124
125	/** @name
126	* States of the VMCS.
127	*
128	* This does not reflect all possible VMCS states but currently only those
129	* needed for maintaining the VMCS consistently even when thread-context hooks
130	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
131	*/
132	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135	/** @} */
136
137	/**
138	* Exception bitmap mask for real-mode guests (real-on-v86).
139	*
140	* We need to intercept all exceptions manually except:
141	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143	* due to bugs in Intel CPUs.
144	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
145	* support.
146	*/
147	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
148	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
149	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
150	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
151	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
153	\| RT_BIT(X86_XCPT_XF))
154
155	/**
156	* Exception bitmap mask for all contributory exceptions.
157	*
158	* Page fault is deliberately excluded here as it's conditional as to whether
159	* it's contributory or benign. Page faults are handled separately.
160	*/
161	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
162	\| RT_BIT(X86_XCPT_DE))
163
164	/** Maximum VM-instruction error number. */
165	#define HMVMX_INSTR_ERROR_MAX 28
166
167	/** Profiling macro. */
168	#ifdef HM_PROFILE_EXIT_DISPATCH
169	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171	#else
172	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174	#endif
175
176	/** Assert that preemption is disabled or covered by thread-context hooks. */
177	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180	/** Assert that we haven't migrated CPUs when thread-context hooks are not
181	* used. */
182	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184	("Illegal migration! Entered on CPU %u Current %u\n", \
185	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187	/** Helper macro for VM-exit handlers called unexpectedly. */
188	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189	do { \
190	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191	return VERR_VMX_UNEXPECTED_EXIT; \
192	} while (0)
193
194
195	/*********************************************************************************************************************************
196	* Structures and Typedefs *
197	*********************************************************************************************************************************/
198	/**
199	* VMX transient state.
200	*
201	* A state structure for holding miscellaneous information across
202	* VMX non-root operation and restored after the transition.
203	*/
204	typedef struct VMXTRANSIENT
205	{
206	/** The host's rflags/eflags. */
207	RTCCUINTREG fEFlags;
208	#if HC_ARCH_BITS == 32
209	uint32_t u32Alignment0;
210	#endif
211	/** The guest's TPR value used for TPR shadowing. */
212	uint8_t u8GuestTpr;
213	/** Alignment. */
214	uint8_t abAlignment0[7];
215
216	/** The basic VM-exit reason. */
217	uint16_t uExitReason;
218	/** Alignment. */
219	uint16_t u16Alignment0;
220	/** The VM-exit interruption error code. */
221	uint32_t uExitIntErrorCode;
222	/** The VM-exit exit code qualification. */
223	uint64_t uExitQualification;
224
225	/** The VM-exit interruption-information field. */
226	uint32_t uExitIntInfo;
227	/** The VM-exit instruction-length field. */
228	uint32_t cbInstr;
229	/** The VM-exit instruction-information field. */
230	union
231	{
232	/** Plain unsigned int representation. */
233	uint32_t u;
234	/** INS and OUTS information. */
235	struct
236	{
237	uint32_t u7Reserved0 : 7;
238	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239	uint32_t u3AddrSize : 3;
240	uint32_t u5Reserved1 : 5;
241	/** The segment register (X86_SREG_XXX). */
242	uint32_t iSegReg : 3;
243	uint32_t uReserved2 : 14;
244	} StrIo;
245	} ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Alignment. */
249	uint8_t abAlignment1[3];
250
251	/** The VM-entry interruption-information field. */
252	uint32_t uEntryIntInfo;
253	/** The VM-entry exception error code field. */
254	uint32_t uEntryXcptErrorCode;
255	/** The VM-entry instruction length field. */
256	uint32_t cbEntryInstr;
257
258	/** IDT-vectoring information field. */
259	uint32_t uIdtVectoringInfo;
260	/** IDT-vectoring error code. */
261	uint32_t uIdtVectoringErrorCode;
262
263	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
264	uint32_t fVmcsFieldsRead;
265
266	/** Whether the guest FPU was active at the time of VM-exit. */
267	bool fWasGuestFPUStateActive;
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting should be setup before VM-entry. */
273	bool fUpdateTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	} VMXTRANSIENT;
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286	/** Pointer to VMX transient state. */
287	typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290	/**
291	* MSR-bitmap read permissions.
292	*/
293	typedef enum VMXMSREXITREAD
294	{
295	/** Reading this MSR causes a VM-exit. */
296	VMXMSREXIT_INTERCEPT_READ = 0xb,
297	/** Reading this MSR does not cause a VM-exit. */
298	VMXMSREXIT_PASSTHRU_READ
299	} VMXMSREXITREAD;
300	/** Pointer to MSR-bitmap read permissions. */
301	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303	/**
304	* MSR-bitmap write permissions.
305	*/
306	typedef enum VMXMSREXITWRITE
307	{
308	/** Writing to this MSR causes a VM-exit. */
309	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310	/** Writing to this MSR does not cause a VM-exit. */
311	VMXMSREXIT_PASSTHRU_WRITE
312	} VMXMSREXITWRITE;
313	/** Pointer to MSR-bitmap write permissions. */
314	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317	/**
318	* VMX VM-exit handler.
319	*
320	* @returns Strict VBox status code (i.e. informational status codes too).
321	* @param pVCpu The cross context virtual CPU structure.
322	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
323	* out-of-sync. Make sure to update the required
324	* fields before using them.
325	* @param pVmxTransient Pointer to the VMX-transient structure.
326	*/
327	#ifndef HMVMX_USE_FUNCTION_TABLE
328	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	#else
330	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331	/** Pointer to VM-exit handler. */
332	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333	#endif
334
335	/**
336	* VMX VM-exit handler, non-strict status code.
337	*
338	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339	*
340	* @returns VBox status code, no informational status code returned.
341	* @param pVCpu The cross context virtual CPU structure.
342	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
343	* out-of-sync. Make sure to update the required
344	* fields before using them.
345	* @param pVmxTransient Pointer to the VMX-transient structure.
346	*
347	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348	* use of that status code will be replaced with VINF_EM_SOMETHING
349	* later when switching over to IEM.
350	*/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353	#else
354	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355	#endif
356
357
358	/*********************************************************************************************************************************
359	* Internal Functions *
360	*********************************************************************************************************************************/
361	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366	bool fStepping, uint32_t *puIntState);
367	#if HC_ARCH_BITS == 32
368	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369	#endif
370	#ifndef HMVMX_USE_FUNCTION_TABLE
371	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374	#else
375	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377	#endif
378
379
380	/** @name VM-exit handlers.
381	* @{
382	*/
383	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395	static FNVMXEXITHANDLER hmR0VmxExitHlt;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411	static FNVMXEXITHANDLER hmR0VmxExitMwait;
412	static FNVMXEXITHANDLER hmR0VmxExitMtf;
413	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414	static FNVMXEXITHANDLER hmR0VmxExitPause;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428	/** @} */
429
430	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441	/*********************************************************************************************************************************
442	* Global Variables *
443	*********************************************************************************************************************************/
444	#ifdef HMVMX_USE_FUNCTION_TABLE
445
446	/**
447	* VMX_EXIT dispatch table.
448	*/
449	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450	{
451	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491	/* 40 UNDEFINED */ hmR0VmxExitPause,
492	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
513	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516	};
517	#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519	#ifdef VBOX_STRICT
520	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521	{
522	/* 0 */ "(Not Used)",
523	/* 1 */ "VMCALL executed in VMX root operation.",
524	/* 2 */ "VMCLEAR with invalid physical address.",
525	/* 3 */ "VMCLEAR with VMXON pointer.",
526	/* 4 */ "VMLAUNCH with non-clear VMCS.",
527	/* 5 */ "VMRESUME with non-launched VMCS.",
528	/* 6 */ "VMRESUME after VMXOFF",
529	/* 7 */ "VM-entry with invalid control fields.",
530	/* 8 */ "VM-entry with invalid host state fields.",
531	/* 9 */ "VMPTRLD with invalid physical address.",
532	/* 10 */ "VMPTRLD with VMXON pointer.",
533	/* 11 */ "VMPTRLD with incorrect revision identifier.",
534	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535	/* 13 */ "VMWRITE to read-only VMCS component.",
536	/* 14 */ "(Not Used)",
537	/* 15 */ "VMXON executed in VMX root operation.",
538	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539	/* 17 */ "VM-entry with non-launched executing VMCS.",
540	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541	/* 19 */ "VMCALL with non-clear VMCS.",
542	/* 20 */ "VMCALL with invalid VM-exit control fields.",
543	/* 21 */ "(Not Used)",
544	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546	/* 24 */ "VMCALL with invalid SMM-monitor features.",
547	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548	/* 26 */ "VM-entry with events blocked by MOV SS.",
549	/* 27 */ "(Not Used)",
550	/* 28 */ "Invalid operand to INVEPT/INVVPID."
551	};
552	#endif /* VBOX_STRICT */
553
554
555
556	/**
557	* Updates the VM's last error record.
558	*
559	* If there was a VMX instruction error, reads the error data from the VMCS and
560	* updates VCPU's last error record as well.
561	*
562	* @param pVM The cross context VM structure.
563	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
564	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565	* VERR_VMX_INVALID_VMCS_FIELD.
566	* @param rc The error code.
567	*/
568	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569	{
570	AssertPtr(pVM);
571	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
573	{
574	AssertPtrReturnVoid(pVCpu);
575	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576	}
577	pVM->hm.s.lLastError = rc;
578	}
579
580
581	/**
582	* Reads the VM-entry interruption-information field from the VMCS into the VMX
583	* transient structure.
584	*
585	* @returns VBox status code.
586	* @param pVmxTransient Pointer to the VMX transient structure.
587	*
588	* @remarks No-long-jump zone!!!
589	*/
590	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591	{
592	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593	AssertRCReturn(rc, rc);
594	return VINF_SUCCESS;
595	}
596
597
598	#ifdef VBOX_STRICT
599	/**
600	* Reads the VM-entry exception error code field from the VMCS into
601	* the VMX transient structure.
602	*
603	* @returns VBox status code.
604	* @param pVmxTransient Pointer to the VMX transient structure.
605	*
606	* @remarks No-long-jump zone!!!
607	*/
608	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609	{
610	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611	AssertRCReturn(rc, rc);
612	return VINF_SUCCESS;
613	}
614	#endif /* VBOX_STRICT */
615
616
617	#ifdef VBOX_STRICT
618	/**
619	* Reads the VM-entry exception error code field from the VMCS into
620	* the VMX transient structure.
621	*
622	* @returns VBox status code.
623	* @param pVmxTransient Pointer to the VMX transient structure.
624	*
625	* @remarks No-long-jump zone!!!
626	*/
627	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628	{
629	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630	AssertRCReturn(rc, rc);
631	return VINF_SUCCESS;
632	}
633	#endif /* VBOX_STRICT */
634
635
636	/**
637	* Reads the VM-exit interruption-information field from the VMCS into the VMX
638	* transient structure.
639	*
640	* @returns VBox status code.
641	* @param pVmxTransient Pointer to the VMX transient structure.
642	*/
643	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644	{
645	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646	{
647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648	AssertRCReturn(rc, rc);
649	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650	}
651	return VINF_SUCCESS;
652	}
653
654
655	/**
656	* Reads the VM-exit interruption error code from the VMCS into the VMX
657	* transient structure.
658	*
659	* @returns VBox status code.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665	{
666	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the VM-exit instruction length field from the VMCS into the VMX
676	* transient structure.
677	*
678	* @returns VBox status code.
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684	{
685	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the VM-exit instruction-information field from the VMCS into
695	* the VMX transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*/
700	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701	{
702	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703	{
704	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705	AssertRCReturn(rc, rc);
706	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707	}
708	return VINF_SUCCESS;
709	}
710
711
712	/**
713	* Reads the exit code qualification from the VMCS into the VMX transient
714	* structure.
715	*
716	* @returns VBox status code.
717	* @param pVCpu The cross context virtual CPU structure of the
718	* calling EMT. (Required for the VMCS cache case.)
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724	{
725	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Reads the IDT-vectoring information field from the VMCS into the VMX
735	* transient structure.
736	*
737	* @returns VBox status code.
738	* @param pVmxTransient Pointer to the VMX transient structure.
739	*
740	* @remarks No-long-jump zone!!!
741	*/
742	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743	{
744	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745	{
746	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747	AssertRCReturn(rc, rc);
748	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749	}
750	return VINF_SUCCESS;
751	}
752
753
754	/**
755	* Reads the IDT-vectoring error code from the VMCS into the VMX
756	* transient structure.
757	*
758	* @returns VBox status code.
759	* @param pVmxTransient Pointer to the VMX transient structure.
760	*/
761	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762	{
763	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764	{
765	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766	AssertRCReturn(rc, rc);
767	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768	}
769	return VINF_SUCCESS;
770	}
771
772
773	/**
774	* Enters VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	* @param pVM The cross context VM structure. Can be
778	* NULL, after a resume.
779	* @param HCPhysCpuPage Physical address of the VMXON region.
780	* @param pvCpuPage Pointer to the VMXON region.
781	*/
782	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783	{
784	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786	Assert(pvCpuPage);
787	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789	if (pVM)
790	{
791	/* Write the VMCS revision dword to the VMXON region. */
792	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793	}
794
795	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796	RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798	/* Enable the VMX bit in CR4 if necessary. */
799	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801	/* Enter VMX root mode. */
802	int rc = VMXEnable(HCPhysCpuPage);
803	if (RT_FAILURE(rc))
804	{
805	if (!(uOldCr4 & X86_CR4_VMXE))
806	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808	if (pVM)
809	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810	}
811
812	/* Restore interrupts. */
813	ASMSetFlags(fEFlags);
814	return rc;
815	}
816
817
818	/**
819	* Exits VMX root mode operation on the current CPU.
820	*
821	* @returns VBox status code.
822	*/
823	static int hmR0VmxLeaveRootMode(void)
824	{
825	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828	RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830	/* If we're for some reason not in VMX root mode, then don't leave it. */
831	RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833	int rc;
834	if (uHostCR4 & X86_CR4_VMXE)
835	{
836	/* Exit VMX root mode and clear the VMX bit in CR4. */
837	VMXDisable();
838	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839	rc = VINF_SUCCESS;
840	}
841	else
842	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844	/* Restore interrupts. */
845	ASMSetFlags(fEFlags);
846	return rc;
847	}
848
849
850	/**
851	* Allocates and maps one physically contiguous page. The allocated page is
852	* zero'd out. (Used by various VT-x structures).
853	*
854	* @returns IPRT status code.
855	* @param pMemObj Pointer to the ring-0 memory object.
856	* @param ppVirt Where to store the virtual address of the
857	* allocation.
858	* @param pHCPhys Where to store the physical address of the
859	* allocation.
860	*/
861	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862	{
863	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868	if (RT_FAILURE(rc))
869	return rc;
870	ppVirt = RTR0MemObjAddress(pMemObj);
871	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
872	ASMMemZero32(*ppVirt, PAGE_SIZE);
873	return VINF_SUCCESS;
874	}
875
876
877	/**
878	* Frees and unmaps an allocated physical page.
879	*
880	* @param pMemObj Pointer to the ring-0 memory object.
881	* @param ppVirt Where to re-initialize the virtual address of
882	* allocation as 0.
883	* @param pHCPhys Where to re-initialize the physical address of the
884	* allocation as 0.
885	*/
886	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887	{
888	AssertPtr(pMemObj);
889	AssertPtr(ppVirt);
890	AssertPtr(pHCPhys);
891	if (*pMemObj != NIL_RTR0MEMOBJ)
892	{
893	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
894	AssertRC(rc);
895	*pMemObj = NIL_RTR0MEMOBJ;
896	*ppVirt = 0;
897	*pHCPhys = 0;
898	}
899	}
900
901
902	/**
903	* Worker function to free VT-x related structures.
904	*
905	* @returns IPRT status code.
906	* @param pVM The cross context VM structure.
907	*/
908	static void hmR0VmxStructsFree(PVM pVM)
909	{
910	for (VMCPUID i = 0; i < pVM->cCpus; i++)
911	{
912	PVMCPU pVCpu = &pVM->aCpus[i];
913	AssertPtr(pVCpu);
914
915	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
922	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923	}
924
925	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928	#endif
929	}
930
931
932	/**
933	* Worker function to allocate VT-x related VM structures.
934	*
935	* @returns IPRT status code.
936	* @param pVM The cross context VM structure.
937	*/
938	static int hmR0VmxStructsAlloc(PVM pVM)
939	{
940	/*
941	* Initialize members up-front so we can cleanup properly on allocation failure.
942	*/
943	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946	pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955	#endif
956	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959	for (VMCPUID i = 0; i < pVM->cCpus; i++)
960	{
961	PVMCPU pVCpu = &pVM->aCpus[i];
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
965	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
966	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
967	}
968	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
969	#undef VMXLOCAL_INIT_VM_MEMOBJ
970
971	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
972	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
973	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
974	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
975
976	/*
977	* Allocate all the VT-x structures.
978	*/
979	int rc = VINF_SUCCESS;
980	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
981	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
982	if (RT_FAILURE(rc))
983	goto cleanup;
984	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
985	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
986	#endif
987
988	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
989	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
990	{
991	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
992	&pVM->hm.s.vmx.HCPhysApicAccess);
993	if (RT_FAILURE(rc))
994	goto cleanup;
995	}
996
997	/*
998	* Initialize per-VCPU VT-x structures.
999	*/
1000	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1001	{
1002	PVMCPU pVCpu = &pVM->aCpus[i];
1003	AssertPtr(pVCpu);
1004
1005	/* Allocate the VM control structure (VMCS). */
1006	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1007	if (RT_FAILURE(rc))
1008	goto cleanup;
1009
1010	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
1011	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1012	{
1013	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1014	&pVCpu->hm.s.vmx.HCPhysVirtApic);
1015	if (RT_FAILURE(rc))
1016	goto cleanup;
1017	}
1018
1019	/*
1020	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1021	* transparent accesses of specific MSRs.
1022	*
1023	* If the condition for enabling MSR bitmaps changes here, don't forget to
1024	* update HMAreMsrBitmapsAvailable().
1025	*/
1026	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1027	{
1028	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1029	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1030	if (RT_FAILURE(rc))
1031	goto cleanup;
1032	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1033	}
1034
1035	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1036	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1037	if (RT_FAILURE(rc))
1038	goto cleanup;
1039
1040	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1041	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1042	if (RT_FAILURE(rc))
1043	goto cleanup;
1044	}
1045
1046	return VINF_SUCCESS;
1047
1048	cleanup:
1049	hmR0VmxStructsFree(pVM);
1050	return rc;
1051	}
1052
1053
1054	/**
1055	* Does global VT-x initialization (called during module initialization).
1056	*
1057	* @returns VBox status code.
1058	*/
1059	VMMR0DECL(int) VMXR0GlobalInit(void)
1060	{
1061	#ifdef HMVMX_USE_FUNCTION_TABLE
1062	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1063	# ifdef VBOX_STRICT
1064	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1065	Assert(g_apfnVMExitHandlers[i]);
1066	# endif
1067	#endif
1068	return VINF_SUCCESS;
1069	}
1070
1071
1072	/**
1073	* Does global VT-x termination (called during module termination).
1074	*/
1075	VMMR0DECL(void) VMXR0GlobalTerm()
1076	{
1077	/* Nothing to do currently. */
1078	}
1079
1080
1081	/**
1082	* Sets up and activates VT-x on the current CPU.
1083	*
1084	* @returns VBox status code.
1085	* @param pCpu Pointer to the global CPU info struct.
1086	* @param pVM The cross context VM structure. Can be
1087	* NULL after a host resume operation.
1088	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1089	* fEnabledByHost is @c true).
1090	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1091	* @a fEnabledByHost is @c true).
1092	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1093	* enable VT-x on the host.
1094	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1095	*/
1096	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1097	void *pvMsrs)
1098	{
1099	Assert(pCpu);
1100	Assert(pvMsrs);
1101	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1102
1103	/* Enable VT-x if it's not already enabled by the host. */
1104	if (!fEnabledByHost)
1105	{
1106	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1107	if (RT_FAILURE(rc))
1108	return rc;
1109	}
1110
1111	/*
1112	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1113	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1114	*/
1115	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117	{
1118	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1119	pCpu->fFlushAsidBeforeUse = false;
1120	}
1121	else
1122	pCpu->fFlushAsidBeforeUse = true;
1123
1124	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125	++pCpu->cTlbFlushes;
1126
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* Deactivates VT-x on the current CPU.
1133	*
1134	* @returns VBox status code.
1135	* @param pCpu Pointer to the global CPU info struct.
1136	* @param pvCpuPage Pointer to the VMXON region.
1137	* @param HCPhysCpuPage Physical address of the VMXON region.
1138	*
1139	* @remarks This function should never be called when SUPR0EnableVTx() or
1140	* similar was used to enable VT-x on the host.
1141	*/
1142	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143	{
1144	NOREF(pCpu);
1145	NOREF(pvCpuPage);
1146	NOREF(HCPhysCpuPage);
1147
1148	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1149	return hmR0VmxLeaveRootMode();
1150	}
1151
1152
1153	/**
1154	* Sets the permission bits for the specified MSR in the MSR bitmap.
1155	*
1156	* @param pVCpu The cross context virtual CPU structure.
1157	* @param uMsr The MSR value.
1158	* @param enmRead Whether reading this MSR causes a VM-exit.
1159	* @param enmWrite Whether writing this MSR causes a VM-exit.
1160	*/
1161	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1162	{
1163	int32_t iBit;
1164	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1165
1166	/*
1167	* Layout:
1168	* 0x000 - 0x3ff - Low MSR read bits
1169	* 0x400 - 0x7ff - High MSR read bits
1170	* 0x800 - 0xbff - Low MSR write bits
1171	* 0xc00 - 0xfff - High MSR write bits
1172	*/
1173	if (uMsr <= 0x00001FFF)
1174	iBit = uMsr;
1175	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1176	{
1177	iBit = uMsr - UINT32_C(0xC0000000);
1178	pbMsrBitmap += 0x400;
1179	}
1180	else
1181	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182
1183	Assert(iBit <= 0x1fff);
1184	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1185	ASMBitSet(pbMsrBitmap, iBit);
1186	else
1187	ASMBitClear(pbMsrBitmap, iBit);
1188
1189	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1190	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1191	else
1192	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1193	}
1194
1195
1196	#ifdef VBOX_STRICT
1197	/**
1198	* Gets the permission bits for the specified MSR in the MSR bitmap.
1199	*
1200	* @returns VBox status code.
1201	* @retval VINF_SUCCESS if the specified MSR is found.
1202	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1203	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1204	*
1205	* @param pVCpu The cross context virtual CPU structure.
1206	* @param uMsr The MSR.
1207	* @param penmRead Where to store the read permissions.
1208	* @param penmWrite Where to store the write permissions.
1209	*/
1210	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1211	{
1212	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1213	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1214	int32_t iBit;
1215	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1216
1217	/* See hmR0VmxSetMsrPermission() for the layout. */
1218	if (uMsr <= 0x00001FFF)
1219	iBit = uMsr;
1220	else if ( uMsr >= 0xC0000000
1221	&& uMsr <= 0xC0001FFF)
1222	{
1223	iBit = (uMsr - 0xC0000000);
1224	pbMsrBitmap += 0x400;
1225	}
1226	else
1227	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1228
1229	Assert(iBit <= 0x1fff);
1230	if (ASMBitTest(pbMsrBitmap, iBit))
1231	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1232	else
1233	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1234
1235	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1236	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1237	else
1238	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1239	return VINF_SUCCESS;
1240	}
1241	#endif /* VBOX_STRICT */
1242
1243
1244	/**
1245	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1246	* area.
1247	*
1248	* @returns VBox status code.
1249	* @param pVCpu The cross context virtual CPU structure.
1250	* @param cMsrs The number of MSRs.
1251	*/
1252	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1253	{
1254	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1255	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1256	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1257	{
1258	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1259	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1260	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1261	}
1262
1263	/* Update number of guest MSRs to load/store across the world-switch. */
1264	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1265	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1266
1267	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1268	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1269	AssertRCReturn(rc, rc);
1270
1271	/* Update the VCPU's copy of the MSR count. */
1272	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1273
1274	return VINF_SUCCESS;
1275	}
1276
1277
1278	/**
1279	* Adds a new (or updates the value of an existing) guest/host MSR
1280	* pair to be swapped during the world-switch as part of the
1281	* auto-load/store MSR area in the VMCS.
1282	*
1283	* @returns VBox status code.
1284	* @param pVCpu The cross context virtual CPU structure.
1285	* @param uMsr The MSR.
1286	* @param uGuestMsrValue Value of the guest MSR.
1287	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1288	* necessary.
1289	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1290	* its value was updated. Optional, can be NULL.
1291	*/
1292	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1293	bool *pfAddedAndUpdated)
1294	{
1295	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1296	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1297	uint32_t i;
1298	for (i = 0; i < cMsrs; i++)
1299	{
1300	if (pGuestMsr->u32Msr == uMsr)
1301	break;
1302	pGuestMsr++;
1303	}
1304
1305	bool fAdded = false;
1306	if (i == cMsrs)
1307	{
1308	++cMsrs;
1309	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1310	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1311
1312	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1313	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1314	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1315
1316	fAdded = true;
1317	}
1318
1319	/* Update the MSR values in the auto-load/store MSR area. */
1320	pGuestMsr->u32Msr = uMsr;
1321	pGuestMsr->u64Value = uGuestMsrValue;
1322
1323	/* Create/update the MSR slot in the host MSR area. */
1324	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1325	pHostMsr += i;
1326	pHostMsr->u32Msr = uMsr;
1327
1328	/*
1329	* Update the host MSR only when requested by the caller AND when we're
1330	* adding it to the auto-load/store area. Otherwise, it would have been
1331	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1332	*/
1333	bool fUpdatedMsrValue = false;
1334	if ( fAdded
1335	&& fUpdateHostMsr)
1336	{
1337	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1338	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1339	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1340	fUpdatedMsrValue = true;
1341	}
1342
1343	if (pfAddedAndUpdated)
1344	*pfAddedAndUpdated = fUpdatedMsrValue;
1345	return VINF_SUCCESS;
1346	}
1347
1348
1349	/**
1350	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1351	* auto-load/store MSR area in the VMCS.
1352	*
1353	* @returns VBox status code.
1354	* @param pVCpu The cross context virtual CPU structure.
1355	* @param uMsr The MSR.
1356	*/
1357	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1358	{
1359	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1360	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1361	for (uint32_t i = 0; i < cMsrs; i++)
1362	{
1363	/* Find the MSR. */
1364	if (pGuestMsr->u32Msr == uMsr)
1365	{
1366	/* If it's the last MSR, simply reduce the count. */
1367	if (i == cMsrs - 1)
1368	{
1369	--cMsrs;
1370	break;
1371	}
1372
1373	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1374	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1375	pLastGuestMsr += cMsrs - 1;
1376	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1377	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1378
1379	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1381	pLastHostMsr += cMsrs - 1;
1382	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1383	pHostMsr->u64Value = pLastHostMsr->u64Value;
1384	--cMsrs;
1385	break;
1386	}
1387	pGuestMsr++;
1388	}
1389
1390	/* Update the VMCS if the count changed (meaning the MSR was found). */
1391	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1392	{
1393	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1394	AssertRCReturn(rc, rc);
1395
1396	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1397	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1398	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1399
1400	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1401	return VINF_SUCCESS;
1402	}
1403
1404	return VERR_NOT_FOUND;
1405	}
1406
1407
1408	/**
1409	* Checks if the specified guest MSR is part of the auto-load/store area in
1410	* the VMCS.
1411	*
1412	* @returns true if found, false otherwise.
1413	* @param pVCpu The cross context virtual CPU structure.
1414	* @param uMsr The MSR to find.
1415	*/
1416	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1417	{
1418	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1419	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1420
1421	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1422	{
1423	if (pGuestMsr->u32Msr == uMsr)
1424	return true;
1425	}
1426	return false;
1427	}
1428
1429
1430	/**
1431	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1432	*
1433	* @param pVCpu The cross context virtual CPU structure.
1434	*
1435	* @remarks No-long-jump zone!!!
1436	*/
1437	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1438	{
1439	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1441	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1445	{
1446	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1447
1448	/*
1449	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1450	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1451	*/
1452	if (pHostMsr->u32Msr == MSR_K6_EFER)
1453	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1454	else
1455	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1456	}
1457
1458	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1459	}
1460
1461
1462	/**
1463	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1464	* perform lazy restoration of the host MSRs while leaving VT-x.
1465	*
1466	* @param pVCpu The cross context virtual CPU structure.
1467	*
1468	* @remarks No-long-jump zone!!!
1469	*/
1470	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1471	{
1472	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1473
1474	/*
1475	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1476	*/
1477	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1478	{
1479	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1480	#if HC_ARCH_BITS == 64
1481	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1482	{
1483	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1484	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1485	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1486	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1487	}
1488	#endif
1489	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1490	}
1491	}
1492
1493
1494	/**
1495	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1496	* lazily while leaving VT-x.
1497	*
1498	* @returns true if it does, false otherwise.
1499	* @param pVCpu The cross context virtual CPU structure.
1500	* @param uMsr The MSR to check.
1501	*/
1502	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1503	{
1504	NOREF(pVCpu);
1505	#if HC_ARCH_BITS == 64
1506	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1507	{
1508	switch (uMsr)
1509	{
1510	case MSR_K8_LSTAR:
1511	case MSR_K6_STAR:
1512	case MSR_K8_SF_MASK:
1513	case MSR_K8_KERNEL_GS_BASE:
1514	return true;
1515	}
1516	}
1517	#else
1518	RT_NOREF(pVCpu, uMsr);
1519	#endif
1520	return false;
1521	}
1522
1523
1524	/**
1525	* Saves a set of guest MSRs back into the guest-CPU context.
1526	*
1527	* @param pVCpu The cross context virtual CPU structure.
1528	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1529	* out-of-sync. Make sure to update the required fields
1530	* before using them.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	*/
1534	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1535	{
1536	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1540	{
1541	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1542	#if HC_ARCH_BITS == 64
1543	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1544	{
1545	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1546	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1547	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1548	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1549	}
1550	#else
1551	NOREF(pMixedCtx);
1552	#endif
1553	}
1554	}
1555
1556
1557	/**
1558	* Loads a set of guests MSRs to allow read/passthru to the guest.
1559	*
1560	* The name of this function is slightly confusing. This function does NOT
1561	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1562	* common prefix for functions dealing with "lazy restoration" of the shared
1563	* MSRs.
1564	*
1565	* @param pVCpu The cross context virtual CPU structure.
1566	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1567	* out-of-sync. Make sure to update the required fields
1568	* before using them.
1569	*
1570	* @remarks No-long-jump zone!!!
1571	*/
1572	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1573	{
1574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1575	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1576
1577	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1578	do { \
1579	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1580	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1581	else \
1582	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1583	} while (0)
1584
1585	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1586	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1587	{
1588	#if HC_ARCH_BITS == 64
1589	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1590	{
1591	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1592	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1593	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1594	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1595	}
1596	#else
1597	RT_NOREF(pMixedCtx);
1598	#endif
1599	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1600	}
1601
1602	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1603	}
1604
1605
1606	/**
1607	* Performs lazy restoration of the set of host MSRs if they were previously
1608	* loaded with guest MSR values.
1609	*
1610	* @param pVCpu The cross context virtual CPU structure.
1611	*
1612	* @remarks No-long-jump zone!!!
1613	* @remarks The guest MSRs should have been saved back into the guest-CPU
1614	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1615	*/
1616	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1617	{
1618	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1619	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1620
1621	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1622	{
1623	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1624	#if HC_ARCH_BITS == 64
1625	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1626	{
1627	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1628	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1629	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1630	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1631	}
1632	#endif
1633	}
1634	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1635	}
1636
1637
1638	/**
1639	* Verifies that our cached values of the VMCS controls are all
1640	* consistent with what's actually present in the VMCS.
1641	*
1642	* @returns VBox status code.
1643	* @param pVCpu The cross context virtual CPU structure.
1644	*/
1645	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1646	{
1647	uint32_t u32Val;
1648	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1649	AssertRCReturn(rc, rc);
1650	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1651	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1652
1653	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1654	AssertRCReturn(rc, rc);
1655	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1656	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1657
1658	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1659	AssertRCReturn(rc, rc);
1660	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1661	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1662
1663	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1664	AssertRCReturn(rc, rc);
1665	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1666	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1667
1668	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1669	{
1670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1671	AssertRCReturn(rc, rc);
1672	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1673	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1674	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1675	}
1676
1677	return VINF_SUCCESS;
1678	}
1679
1680
1681	#ifdef VBOX_STRICT
1682	/**
1683	* Verifies that our cached host EFER value has not changed
1684	* since we cached it.
1685	*
1686	* @param pVCpu The cross context virtual CPU structure.
1687	*/
1688	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1689	{
1690	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1691
1692	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1693	{
1694	uint64_t u64Val;
1695	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1696	AssertRC(rc);
1697
1698	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1699	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1700	}
1701	}
1702
1703
1704	/**
1705	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1706	* VMCS are correct.
1707	*
1708	* @param pVCpu The cross context virtual CPU structure.
1709	*/
1710	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1711	{
1712	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1713
1714	/* Verify MSR counts in the VMCS are what we think it should be. */
1715	uint32_t cMsrs;
1716	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1717	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1718
1719	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1720	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1721
1722	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1723	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1724
1725	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1726	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1727	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1728	{
1729	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1730	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1731	pGuestMsr->u32Msr, cMsrs));
1732
1733	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1734	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1735	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1736
1737	/* Verify that the permissions are as expected in the MSR bitmap. */
1738	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1739	{
1740	VMXMSREXITREAD enmRead;
1741	VMXMSREXITWRITE enmWrite;
1742	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1743	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1744	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1745	{
1746	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1747	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1748	}
1749	else
1750	{
1751	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1752	pGuestMsr->u32Msr, cMsrs));
1753	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1754	pGuestMsr->u32Msr, cMsrs));
1755	}
1756	}
1757	}
1758	}
1759	#endif /* VBOX_STRICT */
1760
1761
1762	/**
1763	* Flushes the TLB using EPT.
1764	*
1765	* @returns VBox status code.
1766	* @param pVCpu The cross context virtual CPU structure of the calling
1767	* EMT. Can be NULL depending on @a enmFlush.
1768	* @param enmFlush Type of flush.
1769	*
1770	* @remarks Caller is responsible for making sure this function is called only
1771	* when NestedPaging is supported and providing @a enmFlush that is
1772	* supported by the CPU.
1773	* @remarks Can be called with interrupts disabled.
1774	*/
1775	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1776	{
1777	uint64_t au64Descriptor[2];
1778	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1779	au64Descriptor[0] = 0;
1780	else
1781	{
1782	Assert(pVCpu);
1783	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1784	}
1785	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1786
1787	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1788	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1789	rc));
1790	if ( RT_SUCCESS(rc)
1791	&& pVCpu)
1792	{
1793	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1794	}
1795	}
1796
1797
1798	/**
1799	* Flushes the TLB using VPID.
1800	*
1801	* @returns VBox status code.
1802	* @param pVM The cross context VM structure.
1803	* @param pVCpu The cross context virtual CPU structure of the calling
1804	* EMT. Can be NULL depending on @a enmFlush.
1805	* @param enmFlush Type of flush.
1806	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1807	* on @a enmFlush).
1808	*
1809	* @remarks Can be called with interrupts disabled.
1810	*/
1811	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1812	{
1813	NOREF(pVM);
1814	AssertPtr(pVM);
1815	Assert(pVM->hm.s.vmx.fVpid);
1816
1817	uint64_t au64Descriptor[2];
1818	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1819	{
1820	au64Descriptor[0] = 0;
1821	au64Descriptor[1] = 0;
1822	}
1823	else
1824	{
1825	AssertPtr(pVCpu);
1826	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1827	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1828	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1829	au64Descriptor[1] = GCPtr;
1830	}
1831
1832	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1833	AssertMsg(rc == VINF_SUCCESS,
1834	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1835	if ( RT_SUCCESS(rc)
1836	&& pVCpu)
1837	{
1838	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1839	}
1840	}
1841
1842
1843	/**
1844	* Invalidates a guest page by guest virtual address. Only relevant for
1845	* EPT/VPID, otherwise there is nothing really to invalidate.
1846	*
1847	* @returns VBox status code.
1848	* @param pVM The cross context VM structure.
1849	* @param pVCpu The cross context virtual CPU structure.
1850	* @param GCVirt Guest virtual address of the page to invalidate.
1851	*/
1852	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1853	{
1854	AssertPtr(pVM);
1855	AssertPtr(pVCpu);
1856	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1857
1858	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1859	if (!fFlushPending)
1860	{
1861	/*
1862	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1863	* See @bugref{6043} and @bugref{6177}.
1864	*
1865	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1866	* function maybe called in a loop with individual addresses.
1867	*/
1868	if (pVM->hm.s.vmx.fVpid)
1869	{
1870	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1871	{
1872	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1873	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1874	}
1875	else
1876	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1877	}
1878	else if (pVM->hm.s.fNestedPaging)
1879	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1880	}
1881
1882	return VINF_SUCCESS;
1883	}
1884
1885
1886	/**
1887	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1888	* otherwise there is nothing really to invalidate.
1889	*
1890	* @returns VBox status code.
1891	* @param pVM The cross context VM structure.
1892	* @param pVCpu The cross context virtual CPU structure.
1893	* @param GCPhys Guest physical address of the page to invalidate.
1894	*/
1895	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1896	{
1897	NOREF(pVM); NOREF(GCPhys);
1898	LogFlowFunc(("%RGp\n", GCPhys));
1899
1900	/*
1901	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1902	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1903	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1904	*/
1905	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1906	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1907	return VINF_SUCCESS;
1908	}
1909
1910
1911	/**
1912	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1913	* case where neither EPT nor VPID is supported by the CPU.
1914	*
1915	* @param pVM The cross context VM structure.
1916	* @param pVCpu The cross context virtual CPU structure.
1917	* @param pCpu Pointer to the global HM struct.
1918	*
1919	* @remarks Called with interrupts disabled.
1920	*/
1921	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922	{
1923	AssertPtr(pVCpu);
1924	AssertPtr(pCpu);
1925	NOREF(pVM);
1926
1927	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1928
1929	Assert(pCpu->idCpu != NIL_RTCPUID);
1930	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1931	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1932	pVCpu->hm.s.fForceTLBFlush = false;
1933	return;
1934	}
1935
1936
1937	/**
1938	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1939	*
1940	* @param pVM The cross context VM structure.
1941	* @param pVCpu The cross context virtual CPU structure.
1942	* @param pCpu Pointer to the global HM CPU struct.
1943	* @remarks All references to "ASID" in this function pertains to "VPID" in
1944	* Intel's nomenclature. The reason is, to avoid confusion in compare
1945	* statements since the host-CPU copies are named "ASID".
1946	*
1947	* @remarks Called with interrupts disabled.
1948	*/
1949	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1950	{
1951	#ifdef VBOX_WITH_STATISTICS
1952	bool fTlbFlushed = false;
1953	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1954	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1955	if (!fTlbFlushed) \
1956	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1957	} while (0)
1958	#else
1959	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1960	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1961	#endif
1962
1963	AssertPtr(pVM);
1964	AssertPtr(pCpu);
1965	AssertPtr(pVCpu);
1966	Assert(pCpu->idCpu != NIL_RTCPUID);
1967
1968	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1969	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1970	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1971
1972	/*
1973	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1974	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1975	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1976	*/
1977	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1978	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1979	{
1980	++pCpu->uCurrentAsid;
1981	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1982	{
1983	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1984	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1985	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1986	}
1987
1988	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1989	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1990	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1991
1992	/*
1993	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1994	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1995	*/
1996	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1997	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1998	HMVMX_SET_TAGGED_TLB_FLUSHED();
1999	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2000	}
2001
2002	/* Check for explicit TLB flushes. */
2003	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2004	{
2005	/*
2006	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2007	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2008	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2009	* but not guest-physical mappings.
2010	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2011	*/
2012	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2013	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2014	HMVMX_SET_TAGGED_TLB_FLUSHED();
2015	}
2016
2017	pVCpu->hm.s.fForceTLBFlush = false;
2018	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2019
2020	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2021	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2022	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2023	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2024	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2025	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2026	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2027	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2028	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2029
2030	/* Update VMCS with the VPID. */
2031	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2032	AssertRC(rc);
2033
2034	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2035	}
2036
2037
2038	/**
2039	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2040	*
2041	* @returns VBox status code.
2042	* @param pVM The cross context VM structure.
2043	* @param pVCpu The cross context virtual CPU structure.
2044	* @param pCpu Pointer to the global HM CPU struct.
2045	*
2046	* @remarks Called with interrupts disabled.
2047	*/
2048	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2049	{
2050	AssertPtr(pVM);
2051	AssertPtr(pVCpu);
2052	AssertPtr(pCpu);
2053	Assert(pCpu->idCpu != NIL_RTCPUID);
2054	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2055	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2056
2057	/*
2058	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2059	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2060	*/
2061	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2062	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2063	{
2064	pVCpu->hm.s.fForceTLBFlush = true;
2065	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2066	}
2067
2068	/* Check for explicit TLB flushes. */
2069	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2070	{
2071	pVCpu->hm.s.fForceTLBFlush = true;
2072	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2073	}
2074
2075	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2076	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2077
2078	if (pVCpu->hm.s.fForceTLBFlush)
2079	{
2080	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2081	pVCpu->hm.s.fForceTLBFlush = false;
2082	}
2083	}
2084
2085
2086	/**
2087	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2088	*
2089	* @returns VBox status code.
2090	* @param pVM The cross context VM structure.
2091	* @param pVCpu The cross context virtual CPU structure.
2092	* @param pCpu Pointer to the global HM CPU struct.
2093	*
2094	* @remarks Called with interrupts disabled.
2095	*/
2096	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2097	{
2098	AssertPtr(pVM);
2099	AssertPtr(pVCpu);
2100	AssertPtr(pCpu);
2101	Assert(pCpu->idCpu != NIL_RTCPUID);
2102	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2103	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2104
2105	/*
2106	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2107	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2108	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2109	*/
2110	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2111	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2112	{
2113	pVCpu->hm.s.fForceTLBFlush = true;
2114	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2115	}
2116
2117	/* Check for explicit TLB flushes. */
2118	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2119	{
2120	/*
2121	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2122	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2123	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2124	*/
2125	pVCpu->hm.s.fForceTLBFlush = true;
2126	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2127	}
2128
2129	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2130	if (pVCpu->hm.s.fForceTLBFlush)
2131	{
2132	++pCpu->uCurrentAsid;
2133	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2134	{
2135	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2136	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2137	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2138	}
2139
2140	pVCpu->hm.s.fForceTLBFlush = false;
2141	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2142	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2143	if (pCpu->fFlushAsidBeforeUse)
2144	{
2145	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2146	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2147	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2148	{
2149	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2150	pCpu->fFlushAsidBeforeUse = false;
2151	}
2152	else
2153	{
2154	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2155	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2156	}
2157	}
2158	}
2159
2160	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2161	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2162	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2163	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2164	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2165	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2166	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2167
2168	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2169	AssertRC(rc);
2170	}
2171
2172
2173	/**
2174	* Flushes the guest TLB entry based on CPU capabilities.
2175	*
2176	* @param pVCpu The cross context virtual CPU structure.
2177	* @param pCpu Pointer to the global HM CPU struct.
2178	*/
2179	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2180	{
2181	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2182	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2183	#endif
2184	PVM pVM = pVCpu->CTX_SUFF(pVM);
2185	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2186	{
2187	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2188	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2189	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2190	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2191	default:
2192	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2193	break;
2194	}
2195
2196	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2197	}
2198
2199
2200	/**
2201	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2202	* TLB entries from the host TLB before VM-entry.
2203	*
2204	* @returns VBox status code.
2205	* @param pVM The cross context VM structure.
2206	*/
2207	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2208	{
2209	/*
2210	* Determine optimal flush type for Nested Paging.
2211	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2212	* guest execution (see hmR3InitFinalizeR0()).
2213	*/
2214	if (pVM->hm.s.fNestedPaging)
2215	{
2216	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2217	{
2218	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2219	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2220	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2221	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2222	else
2223	{
2224	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2225	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2227	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228	}
2229
2230	/* Make sure the write-back cacheable memory type for EPT is supported. */
2231	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2232	{
2233	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2235	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236	}
2237
2238	/* EPT requires a page-walk length of 4. */
2239	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2240	{
2241	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2243	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244	}
2245	}
2246	else
2247	{
2248	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2249	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2250	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2251	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2252	}
2253	}
2254
2255	/*
2256	* Determine optimal flush type for VPID.
2257	*/
2258	if (pVM->hm.s.vmx.fVpid)
2259	{
2260	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2261	{
2262	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2263	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2264	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2265	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2266	else
2267	{
2268	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2269	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2270	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2271	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2272	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2273	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274	pVM->hm.s.vmx.fVpid = false;
2275	}
2276	}
2277	else
2278	{
2279	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2280	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2281	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2282	pVM->hm.s.vmx.fVpid = false;
2283	}
2284	}
2285
2286	/*
2287	* Setup the handler for flushing tagged-TLBs.
2288	*/
2289	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2290	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2291	else if (pVM->hm.s.fNestedPaging)
2292	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2293	else if (pVM->hm.s.vmx.fVpid)
2294	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2295	else
2296	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2297	return VINF_SUCCESS;
2298	}
2299
2300
2301	/**
2302	* Sets up pin-based VM-execution controls in the VMCS.
2303	*
2304	* @returns VBox status code.
2305	* @param pVM The cross context VM structure.
2306	* @param pVCpu The cross context virtual CPU structure.
2307	*/
2308	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2309	{
2310	AssertPtr(pVM);
2311	AssertPtr(pVCpu);
2312
2313	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2314	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2315
2316	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2317	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2318
2319	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2320	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2321
2322	/* Enable the VMX preemption timer. */
2323	if (pVM->hm.s.vmx.fUsePreemptTimer)
2324	{
2325	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2326	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2327	}
2328
2329	#if 0
2330	/* Enable posted-interrupt processing. */
2331	if (pVM->hm.s.fPostedIntrs)
2332	{
2333	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2334	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2335	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2336	}
2337	#endif
2338
2339	if ((val & zap) != val)
2340	{
2341	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2342	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2343	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2344	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2345	}
2346
2347	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2348	AssertRCReturn(rc, rc);
2349
2350	pVCpu->hm.s.vmx.u32PinCtls = val;
2351	return rc;
2352	}
2353
2354
2355	/**
2356	* Sets up processor-based VM-execution controls in the VMCS.
2357	*
2358	* @returns VBox status code.
2359	* @param pVM The cross context VM structure.
2360	* @param pVCpu The cross context virtual CPU structure.
2361	*/
2362	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363	{
2364	AssertPtr(pVM);
2365	AssertPtr(pVCpu);
2366
2367	int rc = VERR_INTERNAL_ERROR_5;
2368	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382	{
2383	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386	}
2387
2388	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389	if (!pVM->hm.s.fNestedPaging)
2390	{
2391	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395	}
2396
2397	/* Use TPR shadowing if supported by the CPU. */
2398	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399	{
2400	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404	AssertRCReturn(rc, rc);
2405
2406	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407	/* CR8 writes cause a VM-exit based on TPR threshold. */
2408	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410	}
2411	else
2412	{
2413	/*
2414	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415	* Set this control only for 64-bit guests.
2416	*/
2417	if (pVM->hm.s.fAllow64BitGuests)
2418	{
2419	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421	}
2422	}
2423
2424	/* Use MSR-bitmaps if supported by the CPU. */
2425	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426	{
2427	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432	AssertRCReturn(rc, rc);
2433
2434	/*
2435	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436	* automatically using dedicated fields in the VMCS.
2437	*/
2438	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444	#if HC_ARCH_BITS == 64
2445	/*
2446	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447	*/
2448	if (pVM->hm.s.fAllow64BitGuests)
2449	{
2450	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454	}
2455	#endif
2456	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2457	}
2458
2459	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2460	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2461	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2462
2463	if ((val & zap) != val)
2464	{
2465	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2466	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2467	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2468	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2469	}
2470
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2472	AssertRCReturn(rc, rc);
2473
2474	pVCpu->hm.s.vmx.u32ProcCtls = val;
2475
2476	/*
2477	* Secondary processor-based VM-execution controls.
2478	*/
2479	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2480	{
2481	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2482	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2483
2484	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2485	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2486
2487	if (pVM->hm.s.fNestedPaging)
2488	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2489	else
2490	{
2491	/*
2492	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2493	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2494	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2495	*/
2496	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2497	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2498	}
2499
2500	if (pVM->hm.s.vmx.fVpid)
2501	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2502
2503	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2504	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2505
2506	#if 0
2507	if (pVM->hm.s.fVirtApicRegs)
2508	{
2509	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2510	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2511
2512	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2513	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2514	}
2515	#endif
2516
2517	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2518	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2519	* done dynamically. */
2520	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2521	{
2522	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2523	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2524	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2525	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2526	AssertRCReturn(rc, rc);
2527	}
2528
2529	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2530	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2531
2532	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2533	&& pVM->hm.s.vmx.cPleGapTicks
2534	&& pVM->hm.s.vmx.cPleWindowTicks)
2535	{
2536	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2537
2538	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2539	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2540	AssertRCReturn(rc, rc);
2541	}
2542
2543	if ((val & zap) != val)
2544	{
2545	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2546	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2547	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2548	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2549	}
2550
2551	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2552	AssertRCReturn(rc, rc);
2553
2554	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2555	}
2556	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2557	{
2558	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2559	"available\n"));
2560	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2561	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2562	}
2563
2564	return VINF_SUCCESS;
2565	}
2566
2567
2568	/**
2569	* Sets up miscellaneous (everything other than Pin & Processor-based
2570	* VM-execution) control fields in the VMCS.
2571	*
2572	* @returns VBox status code.
2573	* @param pVM The cross context VM structure.
2574	* @param pVCpu The cross context virtual CPU structure.
2575	*/
2576	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2577	{
2578	NOREF(pVM);
2579	AssertPtr(pVM);
2580	AssertPtr(pVCpu);
2581
2582	int rc = VERR_GENERAL_FAILURE;
2583
2584	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2585	#if 0
2586	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2587	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2588	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2589
2590	/*
2591	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2592	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2593	* We thus use the exception bitmap to control it rather than use both.
2594	*/
2595	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2596	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2597
2598	/** @todo Explore possibility of using IO-bitmaps. */
2599	/* All IO & IOIO instructions cause VM-exits. */
2600	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2601	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2602
2603	/* Initialize the MSR-bitmap area. */
2604	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2605	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2606	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2607	AssertRCReturn(rc, rc);
2608	#endif
2609
2610	/* Setup MSR auto-load/store area. */
2611	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2612	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2613	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2615	AssertRCReturn(rc, rc);
2616
2617	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2618	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2619	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2620	AssertRCReturn(rc, rc);
2621
2622	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2623	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2624	AssertRCReturn(rc, rc);
2625
2626	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2627	#if 0
2628	/* Setup debug controls */
2629	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2630	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2631	AssertRCReturn(rc, rc);
2632	#endif
2633
2634	return rc;
2635	}
2636
2637
2638	/**
2639	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2640	*
2641	* @returns VBox status code.
2642	* @param pVM The cross context VM structure.
2643	* @param pVCpu The cross context virtual CPU structure.
2644	*/
2645	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2646	{
2647	AssertPtr(pVM);
2648	AssertPtr(pVCpu);
2649
2650	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2651
2652	/** @todo r=ramshankar: Shouldn't setting up \#UD intercepts be handled by
2653	* hmR0VmxLoadGuestXcptIntercepts()? Why do we check it here? */
2654	uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2655
2656	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2658
2659	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2660	and writes, and because recursive #DBs can cause the CPU hang, we must always
2661	intercept #DB. */
2662	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2663
2664	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665	if (!pVM->hm.s.fNestedPaging)
2666	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2667
2668	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2669	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2670	AssertRCReturn(rc, rc);
2671	return rc;
2672	}
2673
2674
2675	/**
2676	* Sets up the initial guest-state mask. The guest-state mask is consulted
2677	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2678	* for the nested virtualization case (as it would cause a VM-exit).
2679	*
2680	* @param pVCpu The cross context virtual CPU structure.
2681	*/
2682	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2683	{
2684	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2685	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2686	return VINF_SUCCESS;
2687	}
2688
2689
2690	/**
2691	* Does per-VM VT-x initialization.
2692	*
2693	* @returns VBox status code.
2694	* @param pVM The cross context VM structure.
2695	*/
2696	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2697	{
2698	LogFlowFunc(("pVM=%p\n", pVM));
2699
2700	int rc = hmR0VmxStructsAlloc(pVM);
2701	if (RT_FAILURE(rc))
2702	{
2703	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2704	return rc;
2705	}
2706
2707	return VINF_SUCCESS;
2708	}
2709
2710
2711	/**
2712	* Does per-VM VT-x termination.
2713	*
2714	* @returns VBox status code.
2715	* @param pVM The cross context VM structure.
2716	*/
2717	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2718	{
2719	LogFlowFunc(("pVM=%p\n", pVM));
2720
2721	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2722	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2723	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2724	#endif
2725	hmR0VmxStructsFree(pVM);
2726	return VINF_SUCCESS;
2727	}
2728
2729
2730	/**
2731	* Sets up the VM for execution under VT-x.
2732	* This function is only called once per-VM during initialization.
2733	*
2734	* @returns VBox status code.
2735	* @param pVM The cross context VM structure.
2736	*/
2737	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2738	{
2739	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2740	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2741
2742	LogFlowFunc(("pVM=%p\n", pVM));
2743
2744	/*
2745	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2746	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2747	*/
2748	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2749	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2750	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2751	{
2752	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2753	return VERR_INTERNAL_ERROR;
2754	}
2755
2756	/* Initialize these always, see hmR3InitFinalizeR0().*/
2757	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2758	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2759
2760	/* Setup the tagged-TLB flush handlers. */
2761	int rc = hmR0VmxSetupTaggedTlb(pVM);
2762	if (RT_FAILURE(rc))
2763	{
2764	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2765	return rc;
2766	}
2767
2768	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2769	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2770	#if HC_ARCH_BITS == 64
2771	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2772	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2773	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2774	{
2775	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2776	}
2777	#endif
2778
2779	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2780	RTCCUINTREG uHostCR4 = ASMGetCR4();
2781	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2782	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2783
2784	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2785	{
2786	PVMCPU pVCpu = &pVM->aCpus[i];
2787	AssertPtr(pVCpu);
2788	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2789
2790	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2791	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2792
2793	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2794	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2795	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2796
2797	/* Set revision dword at the beginning of the VMCS structure. */
2798	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2799
2800	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2801	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2802	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2803	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2804
2805	/* Load this VMCS as the current VMCS. */
2806	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2807	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2811	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2815	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2819	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2823	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2827	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830	#if HC_ARCH_BITS == 32
2831	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2832	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834	#endif
2835
2836	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2837	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2838	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2839	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2840
2841	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2842
2843	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2844	}
2845
2846	return VINF_SUCCESS;
2847	}
2848
2849
2850	/**
2851	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2852	* the VMCS.
2853	*
2854	* @returns VBox status code.
2855	* @param pVM The cross context VM structure.
2856	* @param pVCpu The cross context virtual CPU structure.
2857	*/
2858	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2859	{
2860	NOREF(pVM); NOREF(pVCpu);
2861
2862	RTCCUINTREG uReg = ASMGetCR0();
2863	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2864	AssertRCReturn(rc, rc);
2865
2866	uReg = ASMGetCR3();
2867	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2868	AssertRCReturn(rc, rc);
2869
2870	uReg = ASMGetCR4();
2871	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2872	AssertRCReturn(rc, rc);
2873	return rc;
2874	}
2875
2876
2877	#if HC_ARCH_BITS == 64
2878	/**
2879	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2880	* requirements. See hmR0VmxSaveHostSegmentRegs().
2881	*/
2882	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2883	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2884	{ \
2885	bool fValidSelector = true; \
2886	if ((selValue) & X86_SEL_LDT) \
2887	{ \
2888	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2889	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2890	} \
2891	if (fValidSelector) \
2892	{ \
2893	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2894	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2895	} \
2896	(selValue) = 0; \
2897	}
2898	#endif
2899
2900
2901	/**
2902	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2903	* the host-state area in the VMCS.
2904	*
2905	* @returns VBox status code.
2906	* @param pVM The cross context VM structure.
2907	* @param pVCpu The cross context virtual CPU structure.
2908	*/
2909	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2910	{
2911	int rc = VERR_INTERNAL_ERROR_5;
2912
2913	#if HC_ARCH_BITS == 64
2914	/*
2915	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917	*
2918	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2919	* Was observed booting Solaris10u10 32-bit guest.
2920	*/
2921	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2922	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2923	{
2924	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2925	pVCpu->idCpu));
2926	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2927	}
2928	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2929	#else
2930	RT_NOREF(pVCpu);
2931	#endif
2932
2933	/*
2934	* Host DS, ES, FS and GS segment registers.
2935	*/
2936	#if HC_ARCH_BITS == 64
2937	RTSEL uSelDS = ASMGetDS();
2938	RTSEL uSelES = ASMGetES();
2939	RTSEL uSelFS = ASMGetFS();
2940	RTSEL uSelGS = ASMGetGS();
2941	#else
2942	RTSEL uSelDS = 0;
2943	RTSEL uSelES = 0;
2944	RTSEL uSelFS = 0;
2945	RTSEL uSelGS = 0;
2946	#endif
2947
2948	/*
2949	* Host CS and SS segment registers.
2950	*/
2951	RTSEL uSelCS = ASMGetCS();
2952	RTSEL uSelSS = ASMGetSS();
2953
2954	/*
2955	* Host TR segment register.
2956	*/
2957	RTSEL uSelTR = ASMGetTR();
2958
2959	#if HC_ARCH_BITS == 64
2960	/*
2961	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2962	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2963	*/
2964	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2965	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2966	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2967	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2968	# undef VMXLOCAL_ADJUST_HOST_SEG
2969	#endif
2970
2971	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2972	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2973	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2974	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2975	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2976	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2977	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2978	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2979	Assert(uSelCS);
2980	Assert(uSelTR);
2981
2982	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2983	#if 0
2984	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2985	Assert(uSelSS != 0);
2986	#endif
2987
2988	/* Write these host selector fields into the host-state area in the VMCS. */
2989	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2990	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2991	#if HC_ARCH_BITS == 64
2992	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2993	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2994	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2995	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2996	#else
2997	NOREF(uSelDS);
2998	NOREF(uSelES);
2999	NOREF(uSelFS);
3000	NOREF(uSelGS);
3001	#endif
3002	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3003	AssertRCReturn(rc, rc);
3004
3005	/*
3006	* Host GDTR and IDTR.
3007	*/
3008	RTGDTR Gdtr;
3009	RTIDTR Idtr;
3010	RT_ZERO(Gdtr);
3011	RT_ZERO(Idtr);
3012	ASMGetGDTR(&Gdtr);
3013	ASMGetIDTR(&Idtr);
3014	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3015	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3016	AssertRCReturn(rc, rc);
3017
3018	#if HC_ARCH_BITS == 64
3019	/*
3020	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3021	* maximum limit (0xffff) on every VM-exit.
3022	*/
3023	if (Gdtr.cbGdt != 0xffff)
3024	{
3025	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3026	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3027	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3028	}
3029
3030	/*
3031	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3032	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3033	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3034	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3035	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3036	* hosts where we are pretty sure it won't cause trouble.
3037	*/
3038	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3039	if (Idtr.cbIdt < 0x0fff)
3040	# else
3041	if (Idtr.cbIdt != 0xffff)
3042	# endif
3043	{
3044	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3045	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3046	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3047	}
3048	#endif
3049
3050	/*
3051	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3052	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3053	*/
3054	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3055	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3056	VERR_VMX_INVALID_HOST_STATE);
3057
3058	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3059	#if HC_ARCH_BITS == 64
3060	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3061
3062	/*
3063	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3064	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3065	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3066	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3067	*
3068	* [1] See Intel spec. 3.5 "System Descriptor Types".
3069	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3070	*/
3071	Assert(pDesc->System.u4Type == 11);
3072	if ( pDesc->System.u16LimitLow != 0x67
3073	\|\| pDesc->System.u4LimitHigh)
3074	{
3075	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3076	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3077	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3078	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3079	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3080
3081	/* Store the GDTR here as we need it while restoring TR. */
3082	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3083	}
3084	#else
3085	NOREF(pVM);
3086	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3087	#endif
3088	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3089	AssertRCReturn(rc, rc);
3090
3091	/*
3092	* Host FS base and GS base.
3093	*/
3094	#if HC_ARCH_BITS == 64
3095	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3096	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3097	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3098	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3099	AssertRCReturn(rc, rc);
3100
3101	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3102	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3103	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3104	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3105	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3106	#endif
3107	return rc;
3108	}
3109
3110
3111	/**
3112	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3113	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3114	* the host after every successful VM-exit.
3115	*
3116	* @returns VBox status code.
3117	* @param pVM The cross context VM structure.
3118	* @param pVCpu The cross context virtual CPU structure.
3119	*
3120	* @remarks No-long-jump zone!!!
3121	*/
3122	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3123	{
3124	NOREF(pVM);
3125
3126	AssertPtr(pVCpu);
3127	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3128
3129	/*
3130	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3131	* rather than swapping them on every VM-entry.
3132	*/
3133	hmR0VmxLazySaveHostMsrs(pVCpu);
3134
3135	/*
3136	* Host Sysenter MSRs.
3137	*/
3138	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3139	#if HC_ARCH_BITS == 32
3140	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3141	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3142	#else
3143	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3144	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3145	#endif
3146	AssertRCReturn(rc, rc);
3147
3148	/*
3149	* Host EFER MSR.
3150	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3151	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3152	*/
3153	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3154	{
3155	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3156	AssertRCReturn(rc, rc);
3157	}
3158
3159	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3160	* hmR0VmxLoadGuestExitCtls() !! */
3161
3162	return rc;
3163	}
3164
3165
3166	/**
3167	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3168	*
3169	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3170	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3171	* hmR0VMxLoadGuestEntryCtls().
3172	*
3173	* @returns true if we need to load guest EFER, false otherwise.
3174	* @param pVCpu The cross context virtual CPU structure.
3175	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3176	* out-of-sync. Make sure to update the required fields
3177	* before using them.
3178	*
3179	* @remarks Requires EFER, CR4.
3180	* @remarks No-long-jump zone!!!
3181	*/
3182	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3183	{
3184	#ifdef HMVMX_ALWAYS_SWAP_EFER
3185	return true;
3186	#endif
3187
3188	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3189	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3190	if (CPUMIsGuestInLongMode(pVCpu))
3191	return false;
3192	#endif
3193
3194	PVM pVM = pVCpu->CTX_SUFF(pVM);
3195	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3196	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3197
3198	/*
3199	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3200	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3201	*/
3202	if ( CPUMIsGuestInLongMode(pVCpu)
3203	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3204	{
3205	return true;
3206	}
3207
3208	/*
3209	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3210	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3211	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3212	*/
3213	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3214	&& (pMixedCtx->cr0 & X86_CR0_PG)
3215	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3216	{
3217	/* Assert that host is PAE capable. */
3218	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3219	return true;
3220	}
3221
3222	/** @todo Check the latest Intel spec. for any other bits,
3223	* like SMEP/SMAP? */
3224	return false;
3225	}
3226
3227
3228	/**
3229	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3230	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3231	* controls".
3232	*
3233	* @returns VBox status code.
3234	* @param pVCpu The cross context virtual CPU structure.
3235	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3236	* out-of-sync. Make sure to update the required fields
3237	* before using them.
3238	*
3239	* @remarks Requires EFER.
3240	* @remarks No-long-jump zone!!!
3241	*/
3242	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3243	{
3244	int rc = VINF_SUCCESS;
3245	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3246	{
3247	PVM pVM = pVCpu->CTX_SUFF(pVM);
3248	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256	{
3257	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3259	}
3260	else
3261	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266	{
3267	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3269	}
3270
3271	/*
3272	* The following should -not- be set (since we're not in SMM mode):
3273	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275	*/
3276
3277	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280	if ((val & zap) != val)
3281	{
3282	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3283	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3284	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286	}
3287
3288	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3289	AssertRCReturn(rc, rc);
3290
3291	pVCpu->hm.s.vmx.u32EntryCtls = val;
3292	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3293	}
3294	return rc;
3295	}
3296
3297
3298	/**
3299	* Sets up the VM-exit controls in the VMCS.
3300	*
3301	* @returns VBox status code.
3302	* @param pVCpu The cross context virtual CPU structure.
3303	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3304	* out-of-sync. Make sure to update the required fields
3305	* before using them.
3306	*
3307	* @remarks Requires EFER.
3308	*/
3309	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3310	{
3311	NOREF(pMixedCtx);
3312
3313	int rc = VINF_SUCCESS;
3314	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3315	{
3316	PVM pVM = pVCpu->CTX_SUFF(pVM);
3317	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3318	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3319
3320	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3321	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3322
3323	/*
3324	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3325	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3326	*/
3327	#if HC_ARCH_BITS == 64
3328	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3329	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3330	#else
3331	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3332	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3333	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3334	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3335	{
3336	/* The switcher returns to long mode, EFER is managed by the switcher. */
3337	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3338	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3339	}
3340	else
3341	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3342	#endif
3343
3344	/* If the newer VMCS fields for managing EFER exists, use it. */
3345	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3346	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3347	{
3348	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3349	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3350	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3351	}
3352
3353	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3354	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3355
3356	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3357	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3358	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3359
3360	if ( pVM->hm.s.vmx.fUsePreemptTimer
3361	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3362	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3363
3364	if ((val & zap) != val)
3365	{
3366	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3367	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3368	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3369	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3370	}
3371
3372	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3373	AssertRCReturn(rc, rc);
3374
3375	pVCpu->hm.s.vmx.u32ExitCtls = val;
3376	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3377	}
3378	return rc;
3379	}
3380
3381
3382	/**
3383	* Sets the TPR threshold in the VMCS.
3384	*
3385	* @returns VBox status code.
3386	* @param pVCpu The cross context virtual CPU structure.
3387	* @param u32TprThreshold The TPR threshold (task-priority class only).
3388	*/
3389	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3390	{
3391	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3392	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3393	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3394	}
3395
3396
3397	/**
3398	* Loads the guest APIC and related state.
3399	*
3400	* @returns VBox status code.
3401	* @param pVCpu The cross context virtual CPU structure.
3402	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3403	* out-of-sync. Make sure to update the required fields
3404	* before using them.
3405	*/
3406	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3407	{
3408	NOREF(pMixedCtx);
3409
3410	int rc = VINF_SUCCESS;
3411	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3412	{
3413	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3414	&& APICIsEnabled(pVCpu))
3415	{
3416	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3417	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3418	{
3419	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3420
3421	bool fPendingIntr = false;
3422	uint8_t u8Tpr = 0;
3423	uint8_t u8PendingIntr = 0;
3424	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3425	AssertRCReturn(rc, rc);
3426
3427	/*
3428	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3429	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3430	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3431	* the interrupt when we VM-exit for other reasons.
3432	*/
3433	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3434	uint32_t u32TprThreshold = 0;
3435	if (fPendingIntr)
3436	{
3437	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3438	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3439	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3440	if (u8PendingPriority <= u8TprPriority)
3441	u32TprThreshold = u8PendingPriority;
3442	else
3443	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3444	}
3445
3446	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3447	AssertRCReturn(rc, rc);
3448	}
3449
3450	#ifndef IEM_VERIFICATION_MODE_FULL
3451	/* Setup the Virtualized APIC accesses. */
3452	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
3453	{
3454	uint64_t u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
3455	if (u64MsrApicBase != pVCpu->hm.s.vmx.u64MsrApicBase)
3456	{
3457	PVM pVM = pVCpu->CTX_SUFF(pVM);
3458	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
3459	RTGCPHYS GCPhysApicBase;
3460	GCPhysApicBase = u64MsrApicBase;
3461	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3462
3463	/* Unalias any existing mapping. */
3464	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
3465	AssertRCReturn(rc, rc);
3466
3467	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
3468	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
3469	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
3470	AssertRCReturn(rc, rc);
3471
3472	/* Update VMX's cache of the APIC base. */
3473	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
3474	}
3475	}
3476	#endif /* !IEM_VERIFICATION_MODE_FULL */
3477	}
3478	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3479	}
3480
3481	return rc;
3482	}
3483
3484
3485	/**
3486	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3487	*
3488	* @returns Guest's interruptibility-state.
3489	* @param pVCpu The cross context virtual CPU structure.
3490	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3491	* out-of-sync. Make sure to update the required fields
3492	* before using them.
3493	*
3494	* @remarks No-long-jump zone!!!
3495	*/
3496	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3497	{
3498	/*
3499	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3500	*/
3501	uint32_t uIntrState = 0;
3502	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3503	{
3504	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3505	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3506	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3507	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3508	{
3509	if (pMixedCtx->eflags.Bits.u1IF)
3510	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3511	else
3512	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3513	}
3514	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3515	{
3516	/*
3517	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3518	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3519	*/
3520	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3521	}
3522	}
3523
3524	/*
3525	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3526	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3527	* setting this would block host-NMIs and IRET will not clear the blocking.
3528	*
3529	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3530	*/
3531	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3532	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3533	{
3534	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3535	}
3536
3537	return uIntrState;
3538	}
3539
3540
3541	/**
3542	* Loads the guest's interruptibility-state into the guest-state area in the
3543	* VMCS.
3544	*
3545	* @returns VBox status code.
3546	* @param pVCpu The cross context virtual CPU structure.
3547	* @param uIntrState The interruptibility-state to set.
3548	*/
3549	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3550	{
3551	NOREF(pVCpu);
3552	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3553	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3554	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3555	AssertRC(rc);
3556	return rc;
3557	}
3558
3559
3560	/**
3561	* Loads the exception intercepts required for guest execution in the VMCS.
3562	*
3563	* @returns VBox status code.
3564	* @param pVCpu The cross context virtual CPU structure.
3565	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3566	* out-of-sync. Make sure to update the required fields
3567	* before using them.
3568	*/
3569	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3570	{
3571	NOREF(pMixedCtx);
3572	int rc = VINF_SUCCESS;
3573	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3574	{
3575	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3576	if (pVCpu->hm.s.fGIMTrapXcptUD)
3577	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3578	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3579	else
3580	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3581	#endif
3582
3583	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3584	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3585
3586	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3587	AssertRCReturn(rc, rc);
3588
3589	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3590	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3591	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3592	}
3593	return rc;
3594	}
3595
3596
3597	/**
3598	* Loads the guest's RIP into the guest-state area in the VMCS.
3599	*
3600	* @returns VBox status code.
3601	* @param pVCpu The cross context virtual CPU structure.
3602	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3603	* out-of-sync. Make sure to update the required fields
3604	* before using them.
3605	*
3606	* @remarks No-long-jump zone!!!
3607	*/
3608	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3609	{
3610	int rc = VINF_SUCCESS;
3611	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3612	{
3613	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3614	AssertRCReturn(rc, rc);
3615
3616	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3617	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3618	HMCPU_CF_VALUE(pVCpu)));
3619	}
3620	return rc;
3621	}
3622
3623
3624	/**
3625	* Loads the guest's RSP into the guest-state area in the VMCS.
3626	*
3627	* @returns VBox status code.
3628	* @param pVCpu The cross context virtual CPU structure.
3629	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3630	* out-of-sync. Make sure to update the required fields
3631	* before using them.
3632	*
3633	* @remarks No-long-jump zone!!!
3634	*/
3635	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3636	{
3637	int rc = VINF_SUCCESS;
3638	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3639	{
3640	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3641	AssertRCReturn(rc, rc);
3642
3643	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3644	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3645	}
3646	return rc;
3647	}
3648
3649
3650	/**
3651	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3652	*
3653	* @returns VBox status code.
3654	* @param pVCpu The cross context virtual CPU structure.
3655	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3656	* out-of-sync. Make sure to update the required fields
3657	* before using them.
3658	*
3659	* @remarks No-long-jump zone!!!
3660	*/
3661	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3662	{
3663	int rc = VINF_SUCCESS;
3664	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3665	{
3666	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3667	Let us assert it as such and use 32-bit VMWRITE. */
3668	Assert(!(pMixedCtx->rflags.u64 >> 32));
3669	X86EFLAGS Eflags = pMixedCtx->eflags;
3670	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3671	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3672	* These will never be cleared/set, unless some other part of the VMM
3673	* code is buggy - in which case we're better of finding and fixing
3674	* those bugs than hiding them. */
3675	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3676	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3677	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3678	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3679
3680	/*
3681	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3682	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3683	*/
3684	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3685	{
3686	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3687	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3688	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3689	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3690	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3691	}
3692
3693	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3694	AssertRCReturn(rc, rc);
3695
3696	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3697	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3698	}
3699	return rc;
3700	}
3701
3702
3703	/**
3704	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3705	*
3706	* @returns VBox status code.
3707	* @param pVCpu The cross context virtual CPU structure.
3708	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3709	* out-of-sync. Make sure to update the required fields
3710	* before using them.
3711	*
3712	* @remarks No-long-jump zone!!!
3713	*/
3714	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3715	{
3716	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3717	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3718	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3719	AssertRCReturn(rc, rc);
3720	return rc;
3721	}
3722
3723
3724	/**
3725	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3726	* CR0 is partially shared with the host and we have to consider the FPU bits.
3727	*
3728	* @returns VBox status code.
3729	* @param pVCpu The cross context virtual CPU structure.
3730	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3731	* out-of-sync. Make sure to update the required fields
3732	* before using them.
3733	*
3734	* @remarks No-long-jump zone!!!
3735	*/
3736	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3737	{
3738	/*
3739	* Guest CR0.
3740	* Guest FPU.
3741	*/
3742	int rc = VINF_SUCCESS;
3743	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3744	{
3745	Assert(!(pMixedCtx->cr0 >> 32));
3746	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3747	PVM pVM = pVCpu->CTX_SUFF(pVM);
3748
3749	/* The guest's view (read access) of its CR0 is unblemished. */
3750	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3751	AssertRCReturn(rc, rc);
3752	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3753
3754	/* Setup VT-x's view of the guest CR0. */
3755	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3756	if (pVM->hm.s.fNestedPaging)
3757	{
3758	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3759	{
3760	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3761	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3762	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3763	}
3764	else
3765	{
3766	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3767	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3768	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3769	}
3770
3771	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3772	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3773	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3774
3775	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3776	AssertRCReturn(rc, rc);
3777	}
3778	else
3779	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3780
3781	/*
3782	* Guest FPU bits.
3783	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3784	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3785	*/
3786	u32GuestCR0 \|= X86_CR0_NE;
3787	bool fInterceptNM = false;
3788	if (CPUMIsGuestFPUStateActive(pVCpu))
3789	{
3790	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3791	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3792	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3793	}
3794	else
3795	{
3796	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3797	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3798	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3799	}
3800
3801	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3802	bool fInterceptMF = false;
3803	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3804	fInterceptMF = true;
3805
3806	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3807	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3808	{
3809	Assert(PDMVmmDevHeapIsEnabled(pVM));
3810	Assert(pVM->hm.s.vmx.pRealModeTSS);
3811	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3812	fInterceptNM = true;
3813	fInterceptMF = true;
3814	}
3815	else
3816	{
3817	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3818	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3819	}
3820	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3821
3822	if (fInterceptNM)
3823	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3824	else
3825	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3826
3827	if (fInterceptMF)
3828	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3829	else
3830	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3831
3832	/* Additional intercepts for debugging, define these yourself explicitly. */
3833	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3834	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3835	\| RT_BIT(X86_XCPT_BP)
3836	\| RT_BIT(X86_XCPT_DE)
3837	\| RT_BIT(X86_XCPT_NM)
3838	\| RT_BIT(X86_XCPT_TS)
3839	\| RT_BIT(X86_XCPT_UD)
3840	\| RT_BIT(X86_XCPT_NP)
3841	\| RT_BIT(X86_XCPT_SS)
3842	\| RT_BIT(X86_XCPT_GP)
3843	\| RT_BIT(X86_XCPT_PF)
3844	\| RT_BIT(X86_XCPT_MF)
3845	;
3846	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3847	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3848	#endif
3849
3850	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3851
3852	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3853	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3854	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3855	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3856	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3857	else
3858	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3859
3860	u32GuestCR0 \|= uSetCR0;
3861	u32GuestCR0 &= uZapCR0;
3862	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3863
3864	/* Write VT-x's view of the guest CR0 into the VMCS. */
3865	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3866	AssertRCReturn(rc, rc);
3867	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3868	uZapCR0));
3869
3870	/*
3871	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3872	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3873	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3874	*/
3875	uint32_t u32CR0Mask = 0;
3876	u32CR0Mask = X86_CR0_PE
3877	\| X86_CR0_NE
3878	\| X86_CR0_WP
3879	\| X86_CR0_PG
3880	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3881	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3882	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3883
3884	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3885	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3886	* and @bugref{6944}. */
3887	#if 0
3888	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3889	u32CR0Mask &= ~X86_CR0_PE;
3890	#endif
3891	if (pVM->hm.s.fNestedPaging)
3892	u32CR0Mask &= ~X86_CR0_WP;
3893
3894	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3895	if (fInterceptNM)
3896	{
3897	u32CR0Mask \|= X86_CR0_TS
3898	\| X86_CR0_MP;
3899	}
3900
3901	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3902	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3903	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3904	AssertRCReturn(rc, rc);
3905	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3906
3907	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3908	}
3909	return rc;
3910	}
3911
3912
3913	/**
3914	* Loads the guest control registers (CR3, CR4) into the guest-state area
3915	* in the VMCS.
3916	*
3917	* @returns VBox strict status code.
3918	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3919	* without unrestricted guest access and the VMMDev is not presently
3920	* mapped (e.g. EFI32).
3921	*
3922	* @param pVCpu The cross context virtual CPU structure.
3923	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3924	* out-of-sync. Make sure to update the required fields
3925	* before using them.
3926	*
3927	* @remarks No-long-jump zone!!!
3928	*/
3929	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3930	{
3931	int rc = VINF_SUCCESS;
3932	PVM pVM = pVCpu->CTX_SUFF(pVM);
3933
3934	/*
3935	* Guest CR2.
3936	* It's always loaded in the assembler code. Nothing to do here.
3937	*/
3938
3939	/*
3940	* Guest CR3.
3941	*/
3942	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3943	{
3944	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3945	if (pVM->hm.s.fNestedPaging)
3946	{
3947	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3948
3949	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3950	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3951	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3952	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3953
3954	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3955	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3956	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3957
3958	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3959	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3960	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3961	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3962	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3963	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3964	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3965
3966	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3967	AssertRCReturn(rc, rc);
3968	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3969
3970	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3971	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3972	{
3973	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3974	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3975	{
3976	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3977	AssertRCReturn(rc, rc);
3978	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3979	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3980	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3981	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3982	AssertRCReturn(rc, rc);
3983	}
3984
3985	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3986	have Unrestricted Execution to handle the guest when it's not using paging. */
3987	GCPhysGuestCR3 = pMixedCtx->cr3;
3988	}
3989	else
3990	{
3991	/*
3992	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3993	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3994	* EPT takes care of translating it to host-physical addresses.
3995	*/
3996	RTGCPHYS GCPhys;
3997	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3998
3999	/* We obtain it here every time as the guest could have relocated this PCI region. */
4000	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4001	if (RT_SUCCESS(rc))
4002	{ /* likely */ }
4003	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4004	{
4005	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4006	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4007	}
4008	else
4009	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4010
4011	GCPhysGuestCR3 = GCPhys;
4012	}
4013
4014	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4015	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4016	}
4017	else
4018	{
4019	/* Non-nested paging case, just use the hypervisor's CR3. */
4020	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4021
4022	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4023	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4024	}
4025	AssertRCReturn(rc, rc);
4026
4027	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4028	}
4029
4030	/*
4031	* Guest CR4.
4032	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4033	*/
4034	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4035	{
4036	Assert(!(pMixedCtx->cr4 >> 32));
4037	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4038
4039	/* The guest's view of its CR4 is unblemished. */
4040	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4041	AssertRCReturn(rc, rc);
4042	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4043
4044	/* Setup VT-x's view of the guest CR4. */
4045	/*
4046	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4047	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4048	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4049	*/
4050	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4051	{
4052	Assert(pVM->hm.s.vmx.pRealModeTSS);
4053	Assert(PDMVmmDevHeapIsEnabled(pVM));
4054	u32GuestCR4 &= ~X86_CR4_VME;
4055	}
4056
4057	if (pVM->hm.s.fNestedPaging)
4058	{
4059	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4060	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4061	{
4062	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4063	u32GuestCR4 \|= X86_CR4_PSE;
4064	/* Our identity mapping is a 32-bit page directory. */
4065	u32GuestCR4 &= ~X86_CR4_PAE;
4066	}
4067	/* else use guest CR4.*/
4068	}
4069	else
4070	{
4071	/*
4072	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4073	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4074	*/
4075	switch (pVCpu->hm.s.enmShadowMode)
4076	{
4077	case PGMMODE_REAL: /* Real-mode. */
4078	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4079	case PGMMODE_32_BIT: /* 32-bit paging. */
4080	{
4081	u32GuestCR4 &= ~X86_CR4_PAE;
4082	break;
4083	}
4084
4085	case PGMMODE_PAE: /* PAE paging. */
4086	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4087	{
4088	u32GuestCR4 \|= X86_CR4_PAE;
4089	break;
4090	}
4091
4092	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4093	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4094	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4095	break;
4096	#endif
4097	default:
4098	AssertFailed();
4099	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4100	}
4101	}
4102
4103	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4104	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4105	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4106	u32GuestCR4 \|= uSetCR4;
4107	u32GuestCR4 &= uZapCR4;
4108
4109	/* Write VT-x's view of the guest CR4 into the VMCS. */
4110	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4111	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4112	AssertRCReturn(rc, rc);
4113
4114	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4115	uint32_t u32CR4Mask = X86_CR4_VME
4116	\| X86_CR4_PAE
4117	\| X86_CR4_PGE
4118	\| X86_CR4_PSE
4119	\| X86_CR4_VMXE;
4120	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4121	u32CR4Mask \|= X86_CR4_OSXSAVE;
4122	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4123	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4124	AssertRCReturn(rc, rc);
4125
4126	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4127	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4128
4129	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4130	}
4131	return rc;
4132	}
4133
4134
4135	/**
4136	* Loads the guest debug registers into the guest-state area in the VMCS.
4137	*
4138	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4139	*
4140	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4141	*
4142	* @returns VBox status code.
4143	* @param pVCpu The cross context virtual CPU structure.
4144	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4145	* out-of-sync. Make sure to update the required fields
4146	* before using them.
4147	*
4148	* @remarks No-long-jump zone!!!
4149	*/
4150	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4151	{
4152	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4153	return VINF_SUCCESS;
4154
4155	#ifdef VBOX_STRICT
4156	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4157	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4158	{
4159	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4160	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4161	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4162	}
4163	#endif
4164
4165	int rc;
4166	PVM pVM = pVCpu->CTX_SUFF(pVM);
4167	bool fSteppingDB = false;
4168	bool fInterceptMovDRx = false;
4169	if (pVCpu->hm.s.fSingleInstruction)
4170	{
4171	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4172	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4173	{
4174	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4175	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4176	AssertRCReturn(rc, rc);
4177	Assert(fSteppingDB == false);
4178	}
4179	else
4180	{
4181	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4182	pVCpu->hm.s.fClearTrapFlag = true;
4183	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4184	fSteppingDB = true;
4185	}
4186	}
4187
4188	if ( fSteppingDB
4189	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4190	{
4191	/*
4192	* Use the combined guest and host DRx values found in the hypervisor
4193	* register set because the debugger has breakpoints active or someone
4194	* is single stepping on the host side without a monitor trap flag.
4195	*
4196	* Note! DBGF expects a clean DR6 state before executing guest code.
4197	*/
4198	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4199	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4200	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4201	{
4202	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4203	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4204	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4205	}
4206	else
4207	#endif
4208	if (!CPUMIsHyperDebugStateActive(pVCpu))
4209	{
4210	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4211	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4212	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4213	}
4214
4215	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4216	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4217	AssertRCReturn(rc, rc);
4218
4219	pVCpu->hm.s.fUsingHyperDR7 = true;
4220	fInterceptMovDRx = true;
4221	}
4222	else
4223	{
4224	/*
4225	* If the guest has enabled debug registers, we need to load them prior to
4226	* executing guest code so they'll trigger at the right time.
4227	*/
4228	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4229	{
4230	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4231	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4232	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4233	{
4234	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4235	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4236	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4237	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4238	}
4239	else
4240	#endif
4241	if (!CPUMIsGuestDebugStateActive(pVCpu))
4242	{
4243	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4244	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4245	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4246	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4247	}
4248	Assert(!fInterceptMovDRx);
4249	}
4250	/*
4251	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4252	* must intercept #DB in order to maintain a correct DR6 guest value, and
4253	* because we need to intercept it to prevent nested #DBs from hanging the
4254	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4255	*/
4256	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4257	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4258	&& !CPUMIsGuestDebugStateActive(pVCpu))
4259	#else
4260	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4261	#endif
4262	{
4263	fInterceptMovDRx = true;
4264	}
4265
4266	/* Update guest DR7. */
4267	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4268	AssertRCReturn(rc, rc);
4269
4270	pVCpu->hm.s.fUsingHyperDR7 = false;
4271	}
4272
4273	/*
4274	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4275	*/
4276	if (fInterceptMovDRx)
4277	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4278	else
4279	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4280	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4281	AssertRCReturn(rc, rc);
4282
4283	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4284	return VINF_SUCCESS;
4285	}
4286
4287
4288	#ifdef VBOX_STRICT
4289	/**
4290	* Strict function to validate segment registers.
4291	*
4292	* @remarks ASSUMES CR0 is up to date.
4293	*/
4294	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4295	{
4296	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4297	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4298	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4299	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4300	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4301	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4302	{
4303	/* Protected mode checks */
4304	/* CS */
4305	Assert(pCtx->cs.Attr.n.u1Present);
4306	Assert(!(pCtx->cs.Attr.u & 0xf00));
4307	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4308	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4309	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4310	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4311	\|\| (pCtx->cs.Attr.n.u1Granularity));
4312	/* CS cannot be loaded with NULL in protected mode. */
4313	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4314	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4315	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4316	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4317	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4318	else
4319	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4320	/* SS */
4321	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4322	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4323	if ( !(pCtx->cr0 & X86_CR0_PE)
4324	\|\| pCtx->cs.Attr.n.u4Type == 3)
4325	{
4326	Assert(!pCtx->ss.Attr.n.u2Dpl);
4327	}
4328	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4329	{
4330	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4331	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4332	Assert(pCtx->ss.Attr.n.u1Present);
4333	Assert(!(pCtx->ss.Attr.u & 0xf00));
4334	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4335	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4336	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4337	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4338	\|\| (pCtx->ss.Attr.n.u1Granularity));
4339	}
4340	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4341	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4342	{
4343	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4344	Assert(pCtx->ds.Attr.n.u1Present);
4345	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4346	Assert(!(pCtx->ds.Attr.u & 0xf00));
4347	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4348	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4349	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4350	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4351	\|\| (pCtx->ds.Attr.n.u1Granularity));
4352	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4353	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4354	}
4355	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4356	{
4357	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4358	Assert(pCtx->es.Attr.n.u1Present);
4359	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4360	Assert(!(pCtx->es.Attr.u & 0xf00));
4361	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4362	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4363	\|\| !(pCtx->es.Attr.n.u1Granularity));
4364	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4365	\|\| (pCtx->es.Attr.n.u1Granularity));
4366	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4367	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4368	}
4369	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4370	{
4371	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4372	Assert(pCtx->fs.Attr.n.u1Present);
4373	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4374	Assert(!(pCtx->fs.Attr.u & 0xf00));
4375	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4376	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4377	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4378	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4379	\|\| (pCtx->fs.Attr.n.u1Granularity));
4380	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4381	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4382	}
4383	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4384	{
4385	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4386	Assert(pCtx->gs.Attr.n.u1Present);
4387	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4388	Assert(!(pCtx->gs.Attr.u & 0xf00));
4389	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4390	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4391	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4392	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4393	\|\| (pCtx->gs.Attr.n.u1Granularity));
4394	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4395	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4396	}
4397	/* 64-bit capable CPUs. */
4398	# if HC_ARCH_BITS == 64
4399	Assert(!(pCtx->cs.u64Base >> 32));
4400	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4401	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4402	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4403	# endif
4404	}
4405	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4406	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4407	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4408	{
4409	/* Real and v86 mode checks. */
4410	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4411	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4412	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4413	{
4414	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4415	}
4416	else
4417	{
4418	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4419	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4420	}
4421
4422	/* CS */
4423	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4424	Assert(pCtx->cs.u32Limit == 0xffff);
4425	Assert(u32CSAttr == 0xf3);
4426	/* SS */
4427	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4428	Assert(pCtx->ss.u32Limit == 0xffff);
4429	Assert(u32SSAttr == 0xf3);
4430	/* DS */
4431	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4432	Assert(pCtx->ds.u32Limit == 0xffff);
4433	Assert(u32DSAttr == 0xf3);
4434	/* ES */
4435	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4436	Assert(pCtx->es.u32Limit == 0xffff);
4437	Assert(u32ESAttr == 0xf3);
4438	/* FS */
4439	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4440	Assert(pCtx->fs.u32Limit == 0xffff);
4441	Assert(u32FSAttr == 0xf3);
4442	/* GS */
4443	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4444	Assert(pCtx->gs.u32Limit == 0xffff);
4445	Assert(u32GSAttr == 0xf3);
4446	/* 64-bit capable CPUs. */
4447	# if HC_ARCH_BITS == 64
4448	Assert(!(pCtx->cs.u64Base >> 32));
4449	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4450	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4451	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4452	# endif
4453	}
4454	}
4455	#endif /* VBOX_STRICT */
4456
4457
4458	/**
4459	* Writes a guest segment register into the guest-state area in the VMCS.
4460	*
4461	* @returns VBox status code.
4462	* @param pVCpu The cross context virtual CPU structure.
4463	* @param idxSel Index of the selector in the VMCS.
4464	* @param idxLimit Index of the segment limit in the VMCS.
4465	* @param idxBase Index of the segment base in the VMCS.
4466	* @param idxAccess Index of the access rights of the segment in the VMCS.
4467	* @param pSelReg Pointer to the segment selector.
4468	*
4469	* @remarks No-long-jump zone!!!
4470	*/
4471	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4472	uint32_t idxAccess, PCPUMSELREG pSelReg)
4473	{
4474	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4475	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4476	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4477	AssertRCReturn(rc, rc);
4478
4479	uint32_t u32Access = pSelReg->Attr.u;
4480	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4481	{
4482	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4483	u32Access = 0xf3;
4484	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4485	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4486	}
4487	else
4488	{
4489	/*
4490	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4491	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4492	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4493	* loaded in protected-mode have their attribute as 0.
4494	*/
4495	if (!u32Access)
4496	u32Access = X86DESCATTR_UNUSABLE;
4497	}
4498
4499	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4500	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4501	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4502
4503	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4504	AssertRCReturn(rc, rc);
4505	return rc;
4506	}
4507
4508
4509	/**
4510	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4511	* into the guest-state area in the VMCS.
4512	*
4513	* @returns VBox status code.
4514	* @param pVCpu The cross context virtual CPU structure.
4515	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4516	* out-of-sync. Make sure to update the required fields
4517	* before using them.
4518	*
4519	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4520	* @remarks No-long-jump zone!!!
4521	*/
4522	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4523	{
4524	int rc = VERR_INTERNAL_ERROR_5;
4525	PVM pVM = pVCpu->CTX_SUFF(pVM);
4526
4527	/*
4528	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4529	*/
4530	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4531	{
4532	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4533	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4534	{
4535	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4536	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4537	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4538	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4539	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4540	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4541	}
4542
4543	#ifdef VBOX_WITH_REM
4544	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4545	{
4546	Assert(pVM->hm.s.vmx.pRealModeTSS);
4547	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4548	if ( pVCpu->hm.s.vmx.fWasInRealMode
4549	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4550	{
4551	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4552	in real-mode (e.g. OpenBSD 4.0) */
4553	REMFlushTBs(pVM);
4554	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4555	pVCpu->hm.s.vmx.fWasInRealMode = false;
4556	}
4557	}
4558	#endif
4559	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4560	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4561	AssertRCReturn(rc, rc);
4562	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4563	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4564	AssertRCReturn(rc, rc);
4565	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4566	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4567	AssertRCReturn(rc, rc);
4568	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4569	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4570	AssertRCReturn(rc, rc);
4571	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4572	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4573	AssertRCReturn(rc, rc);
4574	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4575	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4576	AssertRCReturn(rc, rc);
4577
4578	#ifdef VBOX_STRICT
4579	/* Validate. */
4580	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4581	#endif
4582
4583	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4584	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4585	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4586	}
4587
4588	/*
4589	* Guest TR.
4590	*/
4591	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4592	{
4593	/*
4594	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4595	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4596	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4597	*/
4598	uint16_t u16Sel = 0;
4599	uint32_t u32Limit = 0;
4600	uint64_t u64Base = 0;
4601	uint32_t u32AccessRights = 0;
4602
4603	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4604	{
4605	u16Sel = pMixedCtx->tr.Sel;
4606	u32Limit = pMixedCtx->tr.u32Limit;
4607	u64Base = pMixedCtx->tr.u64Base;
4608	u32AccessRights = pMixedCtx->tr.Attr.u;
4609	}
4610	else
4611	{
4612	Assert(pVM->hm.s.vmx.pRealModeTSS);
4613	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4614
4615	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4616	RTGCPHYS GCPhys;
4617	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4618	AssertRCReturn(rc, rc);
4619
4620	X86DESCATTR DescAttr;
4621	DescAttr.u = 0;
4622	DescAttr.n.u1Present = 1;
4623	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4624
4625	u16Sel = 0;
4626	u32Limit = HM_VTX_TSS_SIZE;
4627	u64Base = GCPhys; /* in real-mode phys = virt. */
4628	u32AccessRights = DescAttr.u;
4629	}
4630
4631	/* Validate. */
4632	Assert(!(u16Sel & RT_BIT(2)));
4633	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4634	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4635	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4636	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4637	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4638	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4639	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4640	Assert( (u32Limit & 0xfff) == 0xfff
4641	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4642	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4643	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4644
4645	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4646	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4647	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4648	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4649	AssertRCReturn(rc, rc);
4650
4651	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4652	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4653	}
4654
4655	/*
4656	* Guest GDTR.
4657	*/
4658	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4659	{
4660	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4661	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4662	AssertRCReturn(rc, rc);
4663
4664	/* Validate. */
4665	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4666
4667	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4668	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4669	}
4670
4671	/*
4672	* Guest LDTR.
4673	*/
4674	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4675	{
4676	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4677	uint32_t u32Access = 0;
4678	if (!pMixedCtx->ldtr.Attr.u)
4679	u32Access = X86DESCATTR_UNUSABLE;
4680	else
4681	u32Access = pMixedCtx->ldtr.Attr.u;
4682
4683	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4684	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4685	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4686	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4687	AssertRCReturn(rc, rc);
4688
4689	/* Validate. */
4690	if (!(u32Access & X86DESCATTR_UNUSABLE))
4691	{
4692	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4693	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4694	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4695	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4696	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4697	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4698	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4699	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4700	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4701	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4702	}
4703
4704	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4705	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4706	}
4707
4708	/*
4709	* Guest IDTR.
4710	*/
4711	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4712	{
4713	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4714	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4715	AssertRCReturn(rc, rc);
4716
4717	/* Validate. */
4718	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4719
4720	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4721	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4722	}
4723
4724	return VINF_SUCCESS;
4725	}
4726
4727
4728	/**
4729	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4730	* areas.
4731	*
4732	* These MSRs will automatically be loaded to the host CPU on every successful
4733	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4734	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4735	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4736	*
4737	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4738	*
4739	* @returns VBox status code.
4740	* @param pVCpu The cross context virtual CPU structure.
4741	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4742	* out-of-sync. Make sure to update the required fields
4743	* before using them.
4744	*
4745	* @remarks No-long-jump zone!!!
4746	*/
4747	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4748	{
4749	AssertPtr(pVCpu);
4750	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4751
4752	/*
4753	* MSRs that we use the auto-load/store MSR area in the VMCS.
4754	*/
4755	PVM pVM = pVCpu->CTX_SUFF(pVM);
4756	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4757	{
4758	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4759	#if HC_ARCH_BITS == 32
4760	if (pVM->hm.s.fAllow64BitGuests)
4761	{
4762	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4763	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4764	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4765	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4766	AssertRCReturn(rc, rc);
4767	# ifdef LOG_ENABLED
4768	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4769	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4770	{
4771	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4772	pMsr->u64Value));
4773	}
4774	# endif
4775	}
4776	#endif
4777	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4778	}
4779
4780	/*
4781	* Guest Sysenter MSRs.
4782	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4783	* VM-exits on WRMSRs for these MSRs.
4784	*/
4785	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4786	{
4787	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4788	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4789	}
4790
4791	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4792	{
4793	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4794	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4795	}
4796
4797	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4798	{
4799	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4800	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4801	}
4802
4803	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4804	{
4805	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4806	{
4807	/*
4808	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4809	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4810	*/
4811	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4812	{
4813	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4814	AssertRCReturn(rc,rc);
4815	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4816	}
4817	else
4818	{
4819	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4820	NULL /* pfAddedAndUpdated */);
4821	AssertRCReturn(rc, rc);
4822
4823	/* We need to intercept reads too, see @bugref{7386#c16}. */
4824	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4825	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4826	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4827	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4828	}
4829	}
4830	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4831	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4832	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4833	}
4834
4835	return VINF_SUCCESS;
4836	}
4837
4838
4839	/**
4840	* Loads the guest activity state into the guest-state area in the VMCS.
4841	*
4842	* @returns VBox status code.
4843	* @param pVCpu The cross context virtual CPU structure.
4844	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4845	* out-of-sync. Make sure to update the required fields
4846	* before using them.
4847	*
4848	* @remarks No-long-jump zone!!!
4849	*/
4850	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4851	{
4852	NOREF(pMixedCtx);
4853	/** @todo See if we can make use of other states, e.g.
4854	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4855	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4856	{
4857	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4858	AssertRCReturn(rc, rc);
4859
4860	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4861	}
4862	return VINF_SUCCESS;
4863	}
4864
4865
4866	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4867	/**
4868	* Check if guest state allows safe use of 32-bit switcher again.
4869	*
4870	* Segment bases and protected mode structures must be 32-bit addressable
4871	* because the 32-bit switcher will ignore high dword when writing these VMCS
4872	* fields. See @bugref{8432} for details.
4873	*
4874	* @returns true if safe, false if must continue to use the 64-bit switcher.
4875	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4876	* out-of-sync. Make sure to update the required fields
4877	* before using them.
4878	*
4879	* @remarks No-long-jump zone!!!
4880	*/
4881	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4882	{
4883	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4884	return false;
4885	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4886	return false;
4887	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4888	return false;
4889	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4890	return false;
4891	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4892	return false;
4893	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4894	return false;
4895	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4896	return false;
4897	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4898	return false;
4899	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4900	return false;
4901	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4902	return false;
4903	/* All good, bases are 32-bit. */
4904	return true;
4905	}
4906	#endif
4907
4908
4909	/**
4910	* Sets up the appropriate function to run guest code.
4911	*
4912	* @returns VBox status code.
4913	* @param pVCpu The cross context virtual CPU structure.
4914	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4915	* out-of-sync. Make sure to update the required fields
4916	* before using them.
4917	*
4918	* @remarks No-long-jump zone!!!
4919	*/
4920	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4921	{
4922	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4923	{
4924	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4925	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4926	#endif
4927	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4928	#if HC_ARCH_BITS == 32
4929	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4930	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4931	{
4932	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4933	{
4934	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4935	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4936	\| HM_CHANGED_VMX_ENTRY_CTLS
4937	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4938	}
4939	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4940
4941	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4942	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4943	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4944	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4945	}
4946	#else
4947	/* 64-bit host. */
4948	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4949	#endif
4950	}
4951	else
4952	{
4953	/* Guest is not in long mode, use the 32-bit handler. */
4954	#if HC_ARCH_BITS == 32
4955	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4956	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4957	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4958	{
4959	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4960	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4961	\| HM_CHANGED_VMX_ENTRY_CTLS
4962	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4963	}
4964	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4965	/*
4966	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4967	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4968	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4969	* the much faster 32-bit switcher again.
4970	*/
4971	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4972	{
4973	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4974	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4975	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4976	}
4977	else
4978	{
4979	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4980	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4981	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4982	{
4983	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4984	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4985	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4986	\| HM_CHANGED_VMX_ENTRY_CTLS
4987	\| HM_CHANGED_VMX_EXIT_CTLS
4988	\| HM_CHANGED_HOST_CONTEXT);
4989	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4990	}
4991	}
4992	# else
4993	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4994	# endif
4995	#else
4996	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4997	#endif
4998	}
4999	Assert(pVCpu->hm.s.vmx.pfnStartVM);
5000	return VINF_SUCCESS;
5001	}
5002
5003
5004	/**
5005	* Wrapper for running the guest code in VT-x.
5006	*
5007	* @returns VBox status code, no informational status codes.
5008	* @param pVM The cross context VM structure.
5009	* @param pVCpu The cross context virtual CPU structure.
5010	* @param pCtx Pointer to the guest-CPU context.
5011	*
5012	* @remarks No-long-jump zone!!!
5013	*/
5014	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5015	{
5016	/*
5017	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5018	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5019	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5020	*/
5021	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5022	/** @todo Add stats for resume vs launch. */
5023	#ifdef VBOX_WITH_KERNEL_USING_XMM
5024	int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5025	#else
5026	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5027	#endif
5028	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5029	return rc;
5030	}
5031
5032
5033	/**
5034	* Reports world-switch error and dumps some useful debug info.
5035	*
5036	* @param pVM The cross context VM structure.
5037	* @param pVCpu The cross context virtual CPU structure.
5038	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5039	* @param pCtx Pointer to the guest-CPU context.
5040	* @param pVmxTransient Pointer to the VMX transient structure (only
5041	* exitReason updated).
5042	*/
5043	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5044	{
5045	Assert(pVM);
5046	Assert(pVCpu);
5047	Assert(pCtx);
5048	Assert(pVmxTransient);
5049	HMVMX_ASSERT_PREEMPT_SAFE();
5050
5051	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5052	switch (rcVMRun)
5053	{
5054	case VERR_VMX_INVALID_VMXON_PTR:
5055	AssertFailed();
5056	break;
5057	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5058	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5059	{
5060	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5061	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5062	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5063	AssertRC(rc);
5064
5065	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5066	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5067	Cannot do it here as we may have been long preempted. */
5068
5069	#ifdef VBOX_STRICT
5070	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5071	pVmxTransient->uExitReason));
5072	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5073	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5074	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5075	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5076	else
5077	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5078	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5079	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5080
5081	/* VMX control bits. */
5082	uint32_t u32Val;
5083	uint64_t u64Val;
5084	RTHCUINTREG uHCReg;
5085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5086	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5088	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5089	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5090	{
5091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5092	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5093	}
5094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5095	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5097	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5098	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5099	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5101	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5103	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5105	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5107	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5109	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5111	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5113	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5115	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5116	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5117	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5119	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5120	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5121	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5122	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5123	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5124	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5125	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5126	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5127	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5128	if (pVM->hm.s.fNestedPaging)
5129	{
5130	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5131	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5132	}
5133
5134	/* Guest bits. */
5135	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5136	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5137	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5138	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5139	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5140	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5141	if (pVM->hm.s.vmx.fVpid)
5142	{
5143	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5144	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5145	}
5146
5147	/* Host bits. */
5148	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5149	Log4(("Host CR0 %#RHr\n", uHCReg));
5150	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5151	Log4(("Host CR3 %#RHr\n", uHCReg));
5152	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5153	Log4(("Host CR4 %#RHr\n", uHCReg));
5154
5155	RTGDTR HostGdtr;
5156	PCX86DESCHC pDesc;
5157	ASMGetGDTR(&HostGdtr);
5158	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5159	Log4(("Host CS %#08x\n", u32Val));
5160	if (u32Val < HostGdtr.cbGdt)
5161	{
5162	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5163	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5164	}
5165
5166	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5167	Log4(("Host DS %#08x\n", u32Val));
5168	if (u32Val < HostGdtr.cbGdt)
5169	{
5170	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5171	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5172	}
5173
5174	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5175	Log4(("Host ES %#08x\n", u32Val));
5176	if (u32Val < HostGdtr.cbGdt)
5177	{
5178	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5179	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5180	}
5181
5182	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5183	Log4(("Host FS %#08x\n", u32Val));
5184	if (u32Val < HostGdtr.cbGdt)
5185	{
5186	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5187	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5188	}
5189
5190	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5191	Log4(("Host GS %#08x\n", u32Val));
5192	if (u32Val < HostGdtr.cbGdt)
5193	{
5194	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5195	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5196	}
5197
5198	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5199	Log4(("Host SS %#08x\n", u32Val));
5200	if (u32Val < HostGdtr.cbGdt)
5201	{
5202	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5203	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5204	}
5205
5206	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5207	Log4(("Host TR %#08x\n", u32Val));
5208	if (u32Val < HostGdtr.cbGdt)
5209	{
5210	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5211	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5212	}
5213
5214	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5215	Log4(("Host TR Base %#RHv\n", uHCReg));
5216	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5217	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5218	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5219	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5220	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5221	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5222	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5223	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5224	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5225	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5226	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5227	Log4(("Host RSP %#RHv\n", uHCReg));
5228	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5229	Log4(("Host RIP %#RHv\n", uHCReg));
5230	# if HC_ARCH_BITS == 64
5231	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5232	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5233	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5234	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5235	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5236	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5237	# endif
5238	#endif /* VBOX_STRICT */
5239	break;
5240	}
5241
5242	default:
5243	/* Impossible */
5244	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5245	break;
5246	}
5247	NOREF(pVM); NOREF(pCtx);
5248	}
5249
5250
5251	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5252	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5253	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5254	#endif
5255	#ifdef VBOX_STRICT
5256	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5257	{
5258	switch (idxField)
5259	{
5260	case VMX_VMCS_GUEST_RIP:
5261	case VMX_VMCS_GUEST_RSP:
5262	case VMX_VMCS_GUEST_SYSENTER_EIP:
5263	case VMX_VMCS_GUEST_SYSENTER_ESP:
5264	case VMX_VMCS_GUEST_GDTR_BASE:
5265	case VMX_VMCS_GUEST_IDTR_BASE:
5266	case VMX_VMCS_GUEST_CS_BASE:
5267	case VMX_VMCS_GUEST_DS_BASE:
5268	case VMX_VMCS_GUEST_ES_BASE:
5269	case VMX_VMCS_GUEST_FS_BASE:
5270	case VMX_VMCS_GUEST_GS_BASE:
5271	case VMX_VMCS_GUEST_SS_BASE:
5272	case VMX_VMCS_GUEST_LDTR_BASE:
5273	case VMX_VMCS_GUEST_TR_BASE:
5274	case VMX_VMCS_GUEST_CR3:
5275	return true;
5276	}
5277	return false;
5278	}
5279
5280	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5281	{
5282	switch (idxField)
5283	{
5284	/* Read-only fields. */
5285	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5286	return true;
5287	}
5288	/* Remaining readable fields should also be writable. */
5289	return hmR0VmxIsValidWriteField(idxField);
5290	}
5291	#endif /* VBOX_STRICT */
5292
5293
5294	/**
5295	* Executes the specified handler in 64-bit mode.
5296	*
5297	* @returns VBox status code (no informational status codes).
5298	* @param pVM The cross context VM structure.
5299	* @param pVCpu The cross context virtual CPU structure.
5300	* @param pCtx Pointer to the guest CPU context.
5301	* @param enmOp The operation to perform.
5302	* @param cParams Number of parameters.
5303	* @param paParam Array of 32-bit parameters.
5304	*/
5305	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5306	uint32_t cParams, uint32_t *paParam)
5307	{
5308	NOREF(pCtx);
5309
5310	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5311	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5312	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5313	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5314
5315	#ifdef VBOX_STRICT
5316	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5317	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5318
5319	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5320	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5321	#endif
5322
5323	/* Disable interrupts. */
5324	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5325
5326	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5327	RTCPUID idHostCpu = RTMpCpuId();
5328	CPUMR0SetLApic(pVCpu, idHostCpu);
5329	#endif
5330
5331	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5332	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5333
5334	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5335	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5336	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5337
5338	/* Leave VMX Root Mode. */
5339	VMXDisable();
5340
5341	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5342
5343	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5344	CPUMSetHyperEIP(pVCpu, enmOp);
5345	for (int i = (int)cParams - 1; i >= 0; i--)
5346	CPUMPushHyper(pVCpu, paParam[i]);
5347
5348	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5349
5350	/* Call the switcher. */
5351	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5352	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5353
5354	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5355	/* Make sure the VMX instructions don't cause #UD faults. */
5356	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5357
5358	/* Re-enter VMX Root Mode */
5359	int rc2 = VMXEnable(HCPhysCpuPage);
5360	if (RT_FAILURE(rc2))
5361	{
5362	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5363	ASMSetFlags(fOldEFlags);
5364	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5365	return rc2;
5366	}
5367
5368	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5369	AssertRC(rc2);
5370	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5371	Assert(!(ASMGetFlags() & X86_EFL_IF));
5372	ASMSetFlags(fOldEFlags);
5373	return rc;
5374	}
5375
5376
5377	/**
5378	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5379	* supporting 64-bit guests.
5380	*
5381	* @returns VBox status code.
5382	* @param fResume Whether to VMLAUNCH or VMRESUME.
5383	* @param pCtx Pointer to the guest-CPU context.
5384	* @param pCache Pointer to the VMCS cache.
5385	* @param pVM The cross context VM structure.
5386	* @param pVCpu The cross context virtual CPU structure.
5387	*/
5388	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5389	{
5390	NOREF(fResume);
5391
5392	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5393	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5394
5395	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5396	pCache->uPos = 1;
5397	pCache->interPD = PGMGetInterPaeCR3(pVM);
5398	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5399	#endif
5400
5401	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5402	pCache->TestIn.HCPhysCpuPage = 0;
5403	pCache->TestIn.HCPhysVmcs = 0;
5404	pCache->TestIn.pCache = 0;
5405	pCache->TestOut.HCPhysVmcs = 0;
5406	pCache->TestOut.pCache = 0;
5407	pCache->TestOut.pCtx = 0;
5408	pCache->TestOut.eflags = 0;
5409	#else
5410	NOREF(pCache);
5411	#endif
5412
5413	uint32_t aParam[10];
5414	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5415	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5416	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5417	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5418	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5419	aParam[5] = 0;
5420	aParam[6] = VM_RC_ADDR(pVM, pVM);
5421	aParam[7] = 0;
5422	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5423	aParam[9] = 0;
5424
5425	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5426	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5427	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5428	#endif
5429	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5430
5431	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5432	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5433	Assert(pCtx->dr[4] == 10);
5434	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5435	#endif
5436
5437	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5438	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5439	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5440	pVCpu->hm.s.vmx.HCPhysVmcs));
5441	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5442	pCache->TestOut.HCPhysVmcs));
5443	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5444	pCache->TestOut.pCache));
5445	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5446	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5447	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5448	pCache->TestOut.pCtx));
5449	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5450	#endif
5451	return rc;
5452	}
5453
5454
5455	/**
5456	* Initialize the VMCS-Read cache.
5457	*
5458	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5459	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5460	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5461	* (those that have a 32-bit FULL & HIGH part).
5462	*
5463	* @returns VBox status code.
5464	* @param pVM The cross context VM structure.
5465	* @param pVCpu The cross context virtual CPU structure.
5466	*/
5467	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5468	{
5469	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5470	{ \
5471	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5472	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5473	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5474	++cReadFields; \
5475	}
5476
5477	AssertPtr(pVM);
5478	AssertPtr(pVCpu);
5479	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5480	uint32_t cReadFields = 0;
5481
5482	/*
5483	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5484	* and serve to indicate exceptions to the rules.
5485	*/
5486
5487	/* Guest-natural selector base fields. */
5488	#if 0
5489	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5491	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5492	#endif
5493	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5494	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5495	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5498	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5499	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5505	#if 0
5506	/* Unused natural width guest-state fields. */
5507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5509	#endif
5510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5511	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5512
5513	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5514	#if 0
5515	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5518	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5519	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5520	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5524	#endif
5525
5526	/* Natural width guest-state fields. */
5527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5528	#if 0
5529	/* Currently unused field. */
5530	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5531	#endif
5532
5533	if (pVM->hm.s.fNestedPaging)
5534	{
5535	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5536	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5537	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5538	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5539	}
5540	else
5541	{
5542	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5543	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5544	}
5545
5546	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5547	return VINF_SUCCESS;
5548	}
5549
5550
5551	/**
5552	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5553	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5554	* darwin, running 64-bit guests).
5555	*
5556	* @returns VBox status code.
5557	* @param pVCpu The cross context virtual CPU structure.
5558	* @param idxField The VMCS field encoding.
5559	* @param u64Val 16, 32 or 64-bit value.
5560	*/
5561	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5562	{
5563	int rc;
5564	switch (idxField)
5565	{
5566	/*
5567	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5568	*/
5569	/* 64-bit Control fields. */
5570	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5571	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5572	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5573	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5574	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5575	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5576	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5577	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5578	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5579	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5580	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5581	case VMX_VMCS64_CTRL_EPTP_FULL:
5582	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5583	/* 64-bit Guest-state fields. */
5584	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5585	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5586	case VMX_VMCS64_GUEST_PAT_FULL:
5587	case VMX_VMCS64_GUEST_EFER_FULL:
5588	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5589	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5590	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5591	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5592	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5593	/* 64-bit Host-state fields. */
5594	case VMX_VMCS64_HOST_PAT_FULL:
5595	case VMX_VMCS64_HOST_EFER_FULL:
5596	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5597	{
5598	rc = VMXWriteVmcs32(idxField, u64Val);
5599	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5600	break;
5601	}
5602
5603	/*
5604	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5605	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5606	*/
5607	/* Natural-width Guest-state fields. */
5608	case VMX_VMCS_GUEST_CR3:
5609	case VMX_VMCS_GUEST_ES_BASE:
5610	case VMX_VMCS_GUEST_CS_BASE:
5611	case VMX_VMCS_GUEST_SS_BASE:
5612	case VMX_VMCS_GUEST_DS_BASE:
5613	case VMX_VMCS_GUEST_FS_BASE:
5614	case VMX_VMCS_GUEST_GS_BASE:
5615	case VMX_VMCS_GUEST_LDTR_BASE:
5616	case VMX_VMCS_GUEST_TR_BASE:
5617	case VMX_VMCS_GUEST_GDTR_BASE:
5618	case VMX_VMCS_GUEST_IDTR_BASE:
5619	case VMX_VMCS_GUEST_RSP:
5620	case VMX_VMCS_GUEST_RIP:
5621	case VMX_VMCS_GUEST_SYSENTER_ESP:
5622	case VMX_VMCS_GUEST_SYSENTER_EIP:
5623	{
5624	if (!(u64Val >> 32))
5625	{
5626	/* If this field is 64-bit, VT-x will zero out the top bits. */
5627	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5628	}
5629	else
5630	{
5631	/* Assert that only the 32->64 switcher case should ever come here. */
5632	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5633	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5634	}
5635	break;
5636	}
5637
5638	default:
5639	{
5640	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5641	rc = VERR_INVALID_PARAMETER;
5642	break;
5643	}
5644	}
5645	AssertRCReturn(rc, rc);
5646	return rc;
5647	}
5648
5649
5650	/**
5651	* Queue up a VMWRITE by using the VMCS write cache.
5652	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5653	*
5654	* @param pVCpu The cross context virtual CPU structure.
5655	* @param idxField The VMCS field encoding.
5656	* @param u64Val 16, 32 or 64-bit value.
5657	*/
5658	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5659	{
5660	AssertPtr(pVCpu);
5661	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5662
5663	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5664	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5665
5666	/* Make sure there are no duplicates. */
5667	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5668	{
5669	if (pCache->Write.aField[i] == idxField)
5670	{
5671	pCache->Write.aFieldVal[i] = u64Val;
5672	return VINF_SUCCESS;
5673	}
5674	}
5675
5676	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5677	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5678	pCache->Write.cValidEntries++;
5679	return VINF_SUCCESS;
5680	}
5681	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5682
5683
5684	/**
5685	* Sets up the usage of TSC-offsetting and updates the VMCS.
5686	*
5687	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5688	* VMX preemption timer.
5689	*
5690	* @returns VBox status code.
5691	* @param pVM The cross context VM structure.
5692	* @param pVCpu The cross context virtual CPU structure.
5693	*
5694	* @remarks No-long-jump zone!!!
5695	*/
5696	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5697	{
5698	int rc;
5699	bool fOffsettedTsc;
5700	bool fParavirtTsc;
5701	if (pVM->hm.s.vmx.fUsePreemptTimer)
5702	{
5703	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5704	&fOffsettedTsc, &fParavirtTsc);
5705
5706	/* Make sure the returned values have sane upper and lower boundaries. */
5707	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5708	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5709	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5710	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5711
5712	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5713	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5714	}
5715	else
5716	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5717
5718	/** @todo later optimize this to be done elsewhere and not before every
5719	* VM-entry. */
5720	if (fParavirtTsc)
5721	{
5722	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5723	information before every VM-entry, hence disable it for performance sake. */
5724	#if 0
5725	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5726	AssertRC(rc);
5727	#endif
5728	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5729	}
5730
5731	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5732	{
5733	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5734	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5735
5736	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5737	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5738	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5739	}
5740	else
5741	{
5742	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5743	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5744	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5745	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5746	}
5747	}
5748
5749
5750	/**
5751	* Determines if an exception is a contributory exception.
5752	*
5753	* Contributory exceptions are ones which can cause double-faults unless the
5754	* original exception was a benign exception. Page-fault is intentionally not
5755	* included here as it's a conditional contributory exception.
5756	*
5757	* @returns true if the exception is contributory, false otherwise.
5758	* @param uVector The exception vector.
5759	*/
5760	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5761	{
5762	switch (uVector)
5763	{
5764	case X86_XCPT_GP:
5765	case X86_XCPT_SS:
5766	case X86_XCPT_NP:
5767	case X86_XCPT_TS:
5768	case X86_XCPT_DE:
5769	return true;
5770	default:
5771	break;
5772	}
5773	return false;
5774	}
5775
5776
5777	/**
5778	* Sets an event as a pending event to be injected into the guest.
5779	*
5780	* @param pVCpu The cross context virtual CPU structure.
5781	* @param u32IntInfo The VM-entry interruption-information field.
5782	* @param cbInstr The VM-entry instruction length in bytes (for software
5783	* interrupts, exceptions and privileged software
5784	* exceptions).
5785	* @param u32ErrCode The VM-entry exception error code.
5786	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5787	* page-fault.
5788	*
5789	* @remarks Statistics counter assumes this is a guest event being injected or
5790	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5791	* always incremented.
5792	*/
5793	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5794	RTGCUINTPTR GCPtrFaultAddress)
5795	{
5796	Assert(!pVCpu->hm.s.Event.fPending);
5797	pVCpu->hm.s.Event.fPending = true;
5798	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5799	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5800	pVCpu->hm.s.Event.cbInstr = cbInstr;
5801	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5802	}
5803
5804
5805	/**
5806	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5807	*
5808	* @param pVCpu The cross context virtual CPU structure.
5809	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5810	* out-of-sync. Make sure to update the required fields
5811	* before using them.
5812	*/
5813	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5814	{
5815	NOREF(pMixedCtx);
5816	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5817	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5818	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5819	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5820	}
5821
5822
5823	/**
5824	* Handle a condition that occurred while delivering an event through the guest
5825	* IDT.
5826	*
5827	* @returns Strict VBox status code (i.e. informational status codes too).
5828	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5829	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5830	* to continue execution of the guest which will delivery the \#DF.
5831	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5832	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5833	*
5834	* @param pVCpu The cross context virtual CPU structure.
5835	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5836	* out-of-sync. Make sure to update the required fields
5837	* before using them.
5838	* @param pVmxTransient Pointer to the VMX transient structure.
5839	*
5840	* @remarks No-long-jump zone!!!
5841	*/
5842	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5843	{
5844	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5845
5846	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5847	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5848
5849	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5850	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5851	{
5852	uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5853	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5854
5855	typedef enum
5856	{
5857	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5858	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5859	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5860	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5861	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5862	} VMXREFLECTXCPT;
5863
5864	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5865	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5866	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5867	{
5868	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5869	{
5870	enmReflect = VMXREFLECTXCPT_XCPT;
5871	#ifdef VBOX_STRICT
5872	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5873	&& uExitVector == X86_XCPT_PF)
5874	{
5875	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5876	}
5877	#endif
5878	if ( uExitVector == X86_XCPT_PF
5879	&& uIdtVector == X86_XCPT_PF)
5880	{
5881	pVmxTransient->fVectoringDoublePF = true;
5882	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5883	}
5884	else if ( uExitVector == X86_XCPT_AC
5885	&& uIdtVector == X86_XCPT_AC)
5886	{
5887	enmReflect = VMXREFLECTXCPT_HANG;
5888	Log4(("IDT: Nested #AC - Bad guest\n"));
5889	}
5890	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5891	&& hmR0VmxIsContributoryXcpt(uExitVector)
5892	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5893	\|\| uIdtVector == X86_XCPT_PF))
5894	{
5895	enmReflect = VMXREFLECTXCPT_DF;
5896	}
5897	else if (uIdtVector == X86_XCPT_DF)
5898	enmReflect = VMXREFLECTXCPT_TF;
5899	}
5900	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5901	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5902	{
5903	/*
5904	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5905	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5906	*/
5907	enmReflect = VMXREFLECTXCPT_XCPT;
5908
5909	if (uExitVector == X86_XCPT_PF)
5910	{
5911	pVmxTransient->fVectoringPF = true;
5912	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5913	}
5914	}
5915	}
5916	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5917	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5918	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5919	{
5920	/*
5921	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5922	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5923	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5924	*/
5925	enmReflect = VMXREFLECTXCPT_XCPT;
5926	}
5927
5928	/*
5929	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5930	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5931	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5932	*
5933	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5934	*/
5935	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5936	&& enmReflect == VMXREFLECTXCPT_XCPT
5937	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5938	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5939	{
5940	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5941	}
5942
5943	switch (enmReflect)
5944	{
5945	case VMXREFLECTXCPT_XCPT:
5946	{
5947	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5948	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5949	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5950
5951	uint32_t u32ErrCode = 0;
5952	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5953	{
5954	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5955	AssertRCReturn(rc2, rc2);
5956	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5957	}
5958
5959	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5960	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5961	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5962	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5963	rcStrict = VINF_SUCCESS;
5964	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5965	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5966
5967	break;
5968	}
5969
5970	case VMXREFLECTXCPT_DF:
5971	{
5972	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5973	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5974	rcStrict = VINF_HM_DOUBLE_FAULT;
5975	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5976	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5977
5978	break;
5979	}
5980
5981	case VMXREFLECTXCPT_TF:
5982	{
5983	rcStrict = VINF_EM_RESET;
5984	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5985	uExitVector));
5986	break;
5987	}
5988
5989	case VMXREFLECTXCPT_HANG:
5990	{
5991	rcStrict = VERR_EM_GUEST_CPU_HANG;
5992	break;
5993	}
5994
5995	default:
5996	Assert(rcStrict == VINF_SUCCESS);
5997	break;
5998	}
5999	}
6000	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6001	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6002	&& uExitVector != X86_XCPT_DF
6003	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6004	{
6005	/*
6006	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6007	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6008	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6009	*/
6010	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6011	{
6012	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6013	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6014	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6015	}
6016	}
6017
6018	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6019	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6020	return rcStrict;
6021	}
6022
6023
6024	/**
6025	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6026	*
6027	* @returns VBox status code.
6028	* @param pVCpu The cross context virtual CPU structure.
6029	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6030	* out-of-sync. Make sure to update the required fields
6031	* before using them.
6032	*
6033	* @remarks No-long-jump zone!!!
6034	*/
6035	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6036	{
6037	NOREF(pMixedCtx);
6038
6039	/*
6040	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6041	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6042	*/
6043	VMMRZCallRing3Disable(pVCpu);
6044	HM_DISABLE_PREEMPT();
6045
6046	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6047	{
6048	uint32_t uVal = 0;
6049	uint32_t uShadow = 0;
6050	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6051	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6052	AssertRCReturn(rc, rc);
6053
6054	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6055	CPUMSetGuestCR0(pVCpu, uVal);
6056	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6057	}
6058
6059	HM_RESTORE_PREEMPT();
6060	VMMRZCallRing3Enable(pVCpu);
6061	return VINF_SUCCESS;
6062	}
6063
6064
6065	/**
6066	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6067	*
6068	* @returns VBox status code.
6069	* @param pVCpu The cross context virtual CPU structure.
6070	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6071	* out-of-sync. Make sure to update the required fields
6072	* before using them.
6073	*
6074	* @remarks No-long-jump zone!!!
6075	*/
6076	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6077	{
6078	NOREF(pMixedCtx);
6079
6080	int rc = VINF_SUCCESS;
6081	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6082	{
6083	uint32_t uVal = 0;
6084	uint32_t uShadow = 0;
6085	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6086	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6087	AssertRCReturn(rc, rc);
6088
6089	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6090	CPUMSetGuestCR4(pVCpu, uVal);
6091	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6092	}
6093	return rc;
6094	}
6095
6096
6097	/**
6098	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6099	*
6100	* @returns VBox status code.
6101	* @param pVCpu The cross context virtual CPU structure.
6102	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6103	* out-of-sync. Make sure to update the required fields
6104	* before using them.
6105	*
6106	* @remarks No-long-jump zone!!!
6107	*/
6108	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6109	{
6110	int rc = VINF_SUCCESS;
6111	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6112	{
6113	uint64_t u64Val = 0;
6114	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6115	AssertRCReturn(rc, rc);
6116
6117	pMixedCtx->rip = u64Val;
6118	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6119	}
6120	return rc;
6121	}
6122
6123
6124	/**
6125	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6126	*
6127	* @returns VBox status code.
6128	* @param pVCpu The cross context virtual CPU structure.
6129	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6130	* out-of-sync. Make sure to update the required fields
6131	* before using them.
6132	*
6133	* @remarks No-long-jump zone!!!
6134	*/
6135	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6136	{
6137	int rc = VINF_SUCCESS;
6138	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6139	{
6140	uint64_t u64Val = 0;
6141	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6142	AssertRCReturn(rc, rc);
6143
6144	pMixedCtx->rsp = u64Val;
6145	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6146	}
6147	return rc;
6148	}
6149
6150
6151	/**
6152	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6153	*
6154	* @returns VBox status code.
6155	* @param pVCpu The cross context virtual CPU structure.
6156	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6157	* out-of-sync. Make sure to update the required fields
6158	* before using them.
6159	*
6160	* @remarks No-long-jump zone!!!
6161	*/
6162	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6163	{
6164	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6165	{
6166	uint32_t uVal = 0;
6167	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6168	AssertRCReturn(rc, rc);
6169
6170	pMixedCtx->eflags.u32 = uVal;
6171	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6172	{
6173	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6174	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6175
6176	pMixedCtx->eflags.Bits.u1VM = 0;
6177	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6178	}
6179
6180	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6181	}
6182	return VINF_SUCCESS;
6183	}
6184
6185
6186	/**
6187	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6188	* guest-CPU context.
6189	*/
6190	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6191	{
6192	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6193	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6194	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6195	return rc;
6196	}
6197
6198
6199	/**
6200	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6201	* from the guest-state area in the VMCS.
6202	*
6203	* @param pVCpu The cross context virtual CPU structure.
6204	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6205	* out-of-sync. Make sure to update the required fields
6206	* before using them.
6207	*
6208	* @remarks No-long-jump zone!!!
6209	*/
6210	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6211	{
6212	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6213	{
6214	uint32_t uIntrState = 0;
6215	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6216	AssertRC(rc);
6217
6218	if (!uIntrState)
6219	{
6220	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6221	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6222
6223	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6224	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6225	}
6226	else
6227	{
6228	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6229	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6230	{
6231	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6232	AssertRC(rc);
6233	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6234	AssertRC(rc);
6235
6236	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6237	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6238	}
6239	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6240	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6241
6242	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6243	{
6244	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6245	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6246	}
6247	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6248	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6249	}
6250
6251	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6252	}
6253	}
6254
6255
6256	/**
6257	* Saves the guest's activity state.
6258	*
6259	* @returns VBox status code.
6260	* @param pVCpu The cross context virtual CPU structure.
6261	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6262	* out-of-sync. Make sure to update the required fields
6263	* before using them.
6264	*
6265	* @remarks No-long-jump zone!!!
6266	*/
6267	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6268	{
6269	NOREF(pMixedCtx);
6270	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6271	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6272	return VINF_SUCCESS;
6273	}
6274
6275
6276	/**
6277	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6278	* the current VMCS into the guest-CPU context.
6279	*
6280	* @returns VBox status code.
6281	* @param pVCpu The cross context virtual CPU structure.
6282	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6283	* out-of-sync. Make sure to update the required fields
6284	* before using them.
6285	*
6286	* @remarks No-long-jump zone!!!
6287	*/
6288	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6289	{
6290	int rc = VINF_SUCCESS;
6291	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6292	{
6293	uint32_t u32Val = 0;
6294	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6295	pMixedCtx->SysEnter.cs = u32Val;
6296	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6297	}
6298
6299	uint64_t u64Val = 0;
6300	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6301	{
6302	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6303	pMixedCtx->SysEnter.eip = u64Val;
6304	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6305	}
6306	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6307	{
6308	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6309	pMixedCtx->SysEnter.esp = u64Val;
6310	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6311	}
6312	return rc;
6313	}
6314
6315
6316	/**
6317	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6318	* the CPU back into the guest-CPU context.
6319	*
6320	* @returns VBox status code.
6321	* @param pVCpu The cross context virtual CPU structure.
6322	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6323	* out-of-sync. Make sure to update the required fields
6324	* before using them.
6325	*
6326	* @remarks No-long-jump zone!!!
6327	*/
6328	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6329	{
6330	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6331	VMMRZCallRing3Disable(pVCpu);
6332	HM_DISABLE_PREEMPT();
6333
6334	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6335	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6336	{
6337	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6338	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6339	}
6340
6341	HM_RESTORE_PREEMPT();
6342	VMMRZCallRing3Enable(pVCpu);
6343
6344	return VINF_SUCCESS;
6345	}
6346
6347
6348	/**
6349	* Saves the auto load/store'd guest MSRs from the current VMCS into
6350	* the guest-CPU context.
6351	*
6352	* @returns VBox status code.
6353	* @param pVCpu The cross context virtual CPU structure.
6354	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6355	* out-of-sync. Make sure to update the required fields
6356	* before using them.
6357	*
6358	* @remarks No-long-jump zone!!!
6359	*/
6360	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6361	{
6362	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6363	return VINF_SUCCESS;
6364
6365	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6366	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6367	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6368	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6369	{
6370	switch (pMsr->u32Msr)
6371	{
6372	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6373	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6374	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6375	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6376	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6377	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6378	break;
6379
6380	default:
6381	{
6382	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6383	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6384	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6385	}
6386	}
6387	}
6388
6389	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6390	return VINF_SUCCESS;
6391	}
6392
6393
6394	/**
6395	* Saves the guest control registers from the current VMCS into the guest-CPU
6396	* context.
6397	*
6398	* @returns VBox status code.
6399	* @param pVCpu The cross context virtual CPU structure.
6400	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6401	* out-of-sync. Make sure to update the required fields
6402	* before using them.
6403	*
6404	* @remarks No-long-jump zone!!!
6405	*/
6406	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6407	{
6408	/* Guest CR0. Guest FPU. */
6409	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6410	AssertRCReturn(rc, rc);
6411
6412	/* Guest CR4. */
6413	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6414	AssertRCReturn(rc, rc);
6415
6416	/* Guest CR2 - updated always during the world-switch or in #PF. */
6417	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6418	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6419	{
6420	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6421	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6422
6423	PVM pVM = pVCpu->CTX_SUFF(pVM);
6424	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6425	\|\| ( pVM->hm.s.fNestedPaging
6426	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6427	{
6428	uint64_t u64Val = 0;
6429	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6430	if (pMixedCtx->cr3 != u64Val)
6431	{
6432	CPUMSetGuestCR3(pVCpu, u64Val);
6433	if (VMMRZCallRing3IsEnabled(pVCpu))
6434	{
6435	PGMUpdateCR3(pVCpu, u64Val);
6436	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6437	}
6438	else
6439	{
6440	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6441	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6442	}
6443	}
6444
6445	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6446	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6447	{
6448	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6449	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6450	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6451	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6452	AssertRCReturn(rc, rc);
6453
6454	if (VMMRZCallRing3IsEnabled(pVCpu))
6455	{
6456	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6457	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6458	}
6459	else
6460	{
6461	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6462	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6463	}
6464	}
6465	}
6466
6467	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6468	}
6469
6470	/*
6471	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6472	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6473	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6474	*
6475	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6476	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6477	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6478	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6479	*
6480	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6481	*/
6482	if (VMMRZCallRing3IsEnabled(pVCpu))
6483	{
6484	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6485	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6486
6487	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6488	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6489
6490	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6491	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6492	}
6493
6494	return rc;
6495	}
6496
6497
6498	/**
6499	* Reads a guest segment register from the current VMCS into the guest-CPU
6500	* context.
6501	*
6502	* @returns VBox status code.
6503	* @param pVCpu The cross context virtual CPU structure.
6504	* @param idxSel Index of the selector in the VMCS.
6505	* @param idxLimit Index of the segment limit in the VMCS.
6506	* @param idxBase Index of the segment base in the VMCS.
6507	* @param idxAccess Index of the access rights of the segment in the VMCS.
6508	* @param pSelReg Pointer to the segment selector.
6509	*
6510	* @remarks No-long-jump zone!!!
6511	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6512	* macro as that takes care of whether to read from the VMCS cache or
6513	* not.
6514	*/
6515	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6516	PCPUMSELREG pSelReg)
6517	{
6518	NOREF(pVCpu);
6519
6520	uint32_t u32Val = 0;
6521	int rc = VMXReadVmcs32(idxSel, &u32Val);
6522	AssertRCReturn(rc, rc);
6523	pSelReg->Sel = (uint16_t)u32Val;
6524	pSelReg->ValidSel = (uint16_t)u32Val;
6525	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6526
6527	rc = VMXReadVmcs32(idxLimit, &u32Val);
6528	AssertRCReturn(rc, rc);
6529	pSelReg->u32Limit = u32Val;
6530
6531	uint64_t u64Val = 0;
6532	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6533	AssertRCReturn(rc, rc);
6534	pSelReg->u64Base = u64Val;
6535
6536	rc = VMXReadVmcs32(idxAccess, &u32Val);
6537	AssertRCReturn(rc, rc);
6538	pSelReg->Attr.u = u32Val;
6539
6540	/*
6541	* If VT-x marks the segment as unusable, most other bits remain undefined:
6542	* - For CS the L, D and G bits have meaning.
6543	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6544	* - For the remaining data segments no bits are defined.
6545	*
6546	* The present bit and the unusable bit has been observed to be set at the
6547	* same time (the selector was supposed to be invalid as we started executing
6548	* a V8086 interrupt in ring-0).
6549	*
6550	* What should be important for the rest of the VBox code, is that the P bit is
6551	* cleared. Some of the other VBox code recognizes the unusable bit, but
6552	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6553	* safe side here, we'll strip off P and other bits we don't care about. If
6554	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6555	*
6556	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6557	*/
6558	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6559	{
6560	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6561
6562	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6563	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6564	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6565
6566	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6567	#ifdef DEBUG_bird
6568	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6569	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6570	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6571	#endif
6572	}
6573	return VINF_SUCCESS;
6574	}
6575
6576
6577	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6578	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6579	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6580	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6581	#else
6582	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6583	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6584	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6585	#endif
6586
6587
6588	/**
6589	* Saves the guest segment registers from the current VMCS into the guest-CPU
6590	* context.
6591	*
6592	* @returns VBox status code.
6593	* @param pVCpu The cross context virtual CPU structure.
6594	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6595	* out-of-sync. Make sure to update the required fields
6596	* before using them.
6597	*
6598	* @remarks No-long-jump zone!!!
6599	*/
6600	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6601	{
6602	/* Guest segment registers. */
6603	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6604	{
6605	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6606	AssertRCReturn(rc, rc);
6607
6608	rc = VMXLOCAL_READ_SEG(CS, cs);
6609	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6610	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6611	rc \|= VMXLOCAL_READ_SEG(ES, es);
6612	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6613	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6614	AssertRCReturn(rc, rc);
6615
6616	/* Restore segment attributes for real-on-v86 mode hack. */
6617	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6618	{
6619	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6620	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6621	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6622	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6623	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6624	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6625	}
6626	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6627	}
6628
6629	return VINF_SUCCESS;
6630	}
6631
6632
6633	/**
6634	* Saves the guest descriptor table registers and task register from the current
6635	* VMCS into the guest-CPU context.
6636	*
6637	* @returns VBox status code.
6638	* @param pVCpu The cross context virtual CPU structure.
6639	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6640	* out-of-sync. Make sure to update the required fields
6641	* before using them.
6642	*
6643	* @remarks No-long-jump zone!!!
6644	*/
6645	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6646	{
6647	int rc = VINF_SUCCESS;
6648
6649	/* Guest LDTR. */
6650	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6651	{
6652	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6653	AssertRCReturn(rc, rc);
6654	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6655	}
6656
6657	/* Guest GDTR. */
6658	uint64_t u64Val = 0;
6659	uint32_t u32Val = 0;
6660	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6661	{
6662	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6663	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6664	pMixedCtx->gdtr.pGdt = u64Val;
6665	pMixedCtx->gdtr.cbGdt = u32Val;
6666	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6667	}
6668
6669	/* Guest IDTR. */
6670	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6671	{
6672	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6673	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6674	pMixedCtx->idtr.pIdt = u64Val;
6675	pMixedCtx->idtr.cbIdt = u32Val;
6676	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6677	}
6678
6679	/* Guest TR. */
6680	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6681	{
6682	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6683	AssertRCReturn(rc, rc);
6684
6685	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6686	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6687	{
6688	rc = VMXLOCAL_READ_SEG(TR, tr);
6689	AssertRCReturn(rc, rc);
6690	}
6691	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6692	}
6693	return rc;
6694	}
6695
6696	#undef VMXLOCAL_READ_SEG
6697
6698
6699	/**
6700	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6701	* context.
6702	*
6703	* @returns VBox status code.
6704	* @param pVCpu The cross context virtual CPU structure.
6705	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6706	* out-of-sync. Make sure to update the required fields
6707	* before using them.
6708	*
6709	* @remarks No-long-jump zone!!!
6710	*/
6711	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6712	{
6713	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6714	{
6715	if (!pVCpu->hm.s.fUsingHyperDR7)
6716	{
6717	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6718	uint32_t u32Val;
6719	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6720	pMixedCtx->dr[7] = u32Val;
6721	}
6722
6723	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6724	}
6725	return VINF_SUCCESS;
6726	}
6727
6728
6729	/**
6730	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6731	*
6732	* @returns VBox status code.
6733	* @param pVCpu The cross context virtual CPU structure.
6734	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6735	* out-of-sync. Make sure to update the required fields
6736	* before using them.
6737	*
6738	* @remarks No-long-jump zone!!!
6739	*/
6740	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6741	{
6742	NOREF(pMixedCtx);
6743
6744	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6745	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6746	return VINF_SUCCESS;
6747	}
6748
6749
6750	/**
6751	* Saves the entire guest state from the currently active VMCS into the
6752	* guest-CPU context.
6753	*
6754	* This essentially VMREADs all guest-data.
6755	*
6756	* @returns VBox status code.
6757	* @param pVCpu The cross context virtual CPU structure.
6758	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6759	* out-of-sync. Make sure to update the required fields
6760	* before using them.
6761	*/
6762	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6763	{
6764	Assert(pVCpu);
6765	Assert(pMixedCtx);
6766
6767	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6768	return VINF_SUCCESS;
6769
6770	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6771	again on the ring-3 callback path, there is no real need to. */
6772	if (VMMRZCallRing3IsEnabled(pVCpu))
6773	VMMR0LogFlushDisable(pVCpu);
6774	else
6775	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6776	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6777
6778	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6779	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6780
6781	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6782	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6783
6784	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6785	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6786
6787	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6788	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6789
6790	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6791	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6792
6793	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6794	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6795
6796	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6797	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6798
6799	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6800	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6801
6802	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6803	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6804
6805	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6806	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6807
6808	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6809	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6810	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6811
6812	if (VMMRZCallRing3IsEnabled(pVCpu))
6813	VMMR0LogFlushEnable(pVCpu);
6814
6815	return VINF_SUCCESS;
6816	}
6817
6818
6819	/**
6820	* Saves basic guest registers needed for IEM instruction execution.
6821	*
6822	* @returns VBox status code (OR-able).
6823	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
6824	* @param pMixedCtx Pointer to the CPU context of the guest.
6825	* @param fMemory Whether the instruction being executed operates on
6826	* memory or not. Only CR0 is synced up if clear.
6827	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6828	*/
6829	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6830	{
6831	/*
6832	* We assume all general purpose registers other than RSP are available.
6833	*
6834	* RIP is a must, as it will be incremented or otherwise changed.
6835	*
6836	* RFLAGS are always required to figure the CPL.
6837	*
6838	* RSP isn't always required, however it's a GPR, so frequently required.
6839	*
6840	* SS and CS are the only segment register needed if IEM doesn't do memory
6841	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6842	*
6843	* CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6844	* be required for memory accesses.
6845	*
6846	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
6847	*/
6848	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6849	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6850	if (fNeedRsp)
6851	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6852	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6853	if (!fMemory)
6854	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6855	else
6856	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6857	AssertRCReturn(rc, rc);
6858	return rc;
6859	}
6860
6861
6862	/**
6863	* Ensures that we've got a complete basic guest-context.
6864	*
6865	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
6866	* is for the interpreter.
6867	*
6868	* @returns VBox status code.
6869	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
6870	* @param pMixedCtx Pointer to the guest-CPU context which may have data
6871	* needing to be synced in.
6872	* @thread EMT(pVCpu)
6873	*/
6874	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6875	{
6876	/* Note! Since this is only applicable to VT-x, the implementation is placed
6877	in the VT-x part of the sources instead of the generic stuff. */
6878	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6879	{
6880	/* For now, imply that the caller might change everything too. */
6881	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6882	return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6883	}
6884	return VINF_SUCCESS;
6885	}
6886
6887
6888	/**
6889	* Check per-VM and per-VCPU force flag actions that require us to go back to
6890	* ring-3 for one reason or another.
6891	*
6892	* @returns Strict VBox status code (i.e. informational status codes too)
6893	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6894	* ring-3.
6895	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6896	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6897	* interrupts)
6898	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6899	* all EMTs to be in ring-3.
6900	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6901	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6902	* to the EM loop.
6903	*
6904	* @param pVM The cross context VM structure.
6905	* @param pVCpu The cross context virtual CPU structure.
6906	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6907	* out-of-sync. Make sure to update the required fields
6908	* before using them.
6909	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
6910	*/
6911	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6912	{
6913	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6914
6915	/*
6916	* Anything pending? Should be more likely than not if we're doing a good job.
6917	*/
6918	if ( !fStepping
6919	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6920	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6921	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6922	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6923	return VINF_SUCCESS;
6924
6925	/* We need the control registers now, make sure the guest-CPU context is updated. */
6926	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6927	AssertRCReturn(rc3, rc3);
6928
6929	/* Pending HM CR3 sync. */
6930	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6931	{
6932	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6933	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6934	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6935	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6936	}
6937
6938	/* Pending HM PAE PDPEs. */
6939	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6940	{
6941	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6942	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6943	}
6944
6945	/* Pending PGM C3 sync. */
6946	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6947	{
6948	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6949	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6950	if (rcStrict2 != VINF_SUCCESS)
6951	{
6952	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6953	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6954	return rcStrict2;
6955	}
6956	}
6957
6958	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6959	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6960	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6961	{
6962	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6963	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6964	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6965	return rc2;
6966	}
6967
6968	/* Pending VM request packets, such as hardware interrupts. */
6969	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6970	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6971	{
6972	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6973	return VINF_EM_PENDING_REQUEST;
6974	}
6975
6976	/* Pending PGM pool flushes. */
6977	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6978	{
6979	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6980	return VINF_PGM_POOL_FLUSH_PENDING;
6981	}
6982
6983	/* Pending DMA requests. */
6984	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6985	{
6986	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6987	return VINF_EM_RAW_TO_R3;
6988	}
6989
6990	return VINF_SUCCESS;
6991	}
6992
6993
6994	/**
6995	* Converts any TRPM trap into a pending HM event. This is typically used when
6996	* entering from ring-3 (not longjmp returns).
6997	*
6998	* @param pVCpu The cross context virtual CPU structure.
6999	*/
7000	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7001	{
7002	Assert(TRPMHasTrap(pVCpu));
7003	Assert(!pVCpu->hm.s.Event.fPending);
7004
7005	uint8_t uVector;
7006	TRPMEVENT enmTrpmEvent;
7007	RTGCUINT uErrCode;
7008	RTGCUINTPTR GCPtrFaultAddress;
7009	uint8_t cbInstr;
7010
7011	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7012	AssertRC(rc);
7013
7014	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7015	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7016	if (enmTrpmEvent == TRPM_TRAP)
7017	{
7018	switch (uVector)
7019	{
7020	case X86_XCPT_NMI:
7021	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7022	break;
7023
7024	case X86_XCPT_BP:
7025	case X86_XCPT_OF:
7026	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7027	break;
7028
7029	case X86_XCPT_PF:
7030	case X86_XCPT_DF:
7031	case X86_XCPT_TS:
7032	case X86_XCPT_NP:
7033	case X86_XCPT_SS:
7034	case X86_XCPT_GP:
7035	case X86_XCPT_AC:
7036	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7037	/* no break! */
7038	default:
7039	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7040	break;
7041	}
7042	}
7043	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7044	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7045	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7046	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7047	else
7048	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7049
7050	rc = TRPMResetTrap(pVCpu);
7051	AssertRC(rc);
7052	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7053	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7054
7055	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7056	}
7057
7058
7059	/**
7060	* Converts the pending HM event into a TRPM trap.
7061	*
7062	* @param pVCpu The cross context virtual CPU structure.
7063	*/
7064	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7065	{
7066	Assert(pVCpu->hm.s.Event.fPending);
7067
7068	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7069	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7070	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7071	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7072
7073	/* If a trap was already pending, we did something wrong! */
7074	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7075
7076	TRPMEVENT enmTrapType;
7077	switch (uVectorType)
7078	{
7079	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7080	enmTrapType = TRPM_HARDWARE_INT;
7081	break;
7082
7083	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7084	enmTrapType = TRPM_SOFTWARE_INT;
7085	break;
7086
7087	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7088	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7089	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7090	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7091	enmTrapType = TRPM_TRAP;
7092	break;
7093
7094	default:
7095	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7096	enmTrapType = TRPM_32BIT_HACK;
7097	break;
7098	}
7099
7100	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7101
7102	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7103	AssertRC(rc);
7104
7105	if (fErrorCodeValid)
7106	TRPMSetErrorCode(pVCpu, uErrorCode);
7107
7108	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7109	&& uVector == X86_XCPT_PF)
7110	{
7111	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7112	}
7113	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7114	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7115	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7116	{
7117	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7118	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7119	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7120	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7121	}
7122
7123	/* Clear any pending events from the VMCS. */
7124	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7125	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7126
7127	/* We're now done converting the pending event. */
7128	pVCpu->hm.s.Event.fPending = false;
7129	}
7130
7131
7132	/**
7133	* Does the necessary state syncing before returning to ring-3 for any reason
7134	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7135	*
7136	* @returns VBox status code.
7137	* @param pVCpu The cross context virtual CPU structure.
7138	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7139	* be out-of-sync. Make sure to update the required
7140	* fields before using them.
7141	* @param fSaveGuestState Whether to save the guest state or not.
7142	*
7143	* @remarks No-long-jmp zone!!!
7144	*/
7145	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7146	{
7147	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7148	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7149
7150	RTCPUID idCpu = RTMpCpuId();
7151	Log4Func(("HostCpuId=%u\n", idCpu));
7152
7153	/*
7154	* !!! IMPORTANT !!!
7155	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7156	*/
7157
7158	/* Save the guest state if necessary. */
7159	if ( fSaveGuestState
7160	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7161	{
7162	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7163	AssertRCReturn(rc, rc);
7164	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7165	}
7166
7167	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7168	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7169	{
7170	if (fSaveGuestState)
7171	{
7172	/* We shouldn't reload CR0 without saving it first. */
7173	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7174	AssertRCReturn(rc, rc);
7175	}
7176	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7177	}
7178
7179	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7180	#ifdef VBOX_STRICT
7181	if (CPUMIsHyperDebugStateActive(pVCpu))
7182	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7183	#endif
7184	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7185	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7186	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7187	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7188
7189	#if HC_ARCH_BITS == 64
7190	/* Restore host-state bits that VT-x only restores partially. */
7191	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7192	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7193	{
7194	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7195	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7196	}
7197	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7198	#endif
7199
7200	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7201	if (pVCpu->hm.s.vmx.fLazyMsrs)
7202	{
7203	/* We shouldn't reload the guest MSRs without saving it first. */
7204	if (!fSaveGuestState)
7205	{
7206	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7207	AssertRCReturn(rc, rc);
7208	}
7209	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7210	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7211	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7212	}
7213
7214	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7215	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7216
7217	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7218	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7219	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7220	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7221	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7222	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7223	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7224	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7225
7226	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7227
7228	/** @todo This partially defeats the purpose of having preemption hooks.
7229	* The problem is, deregistering the hooks should be moved to a place that
7230	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7231	* context.
7232	*/
7233	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7234	{
7235	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7236	AssertRCReturn(rc, rc);
7237
7238	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7239	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7240	}
7241	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7242	NOREF(idCpu);
7243
7244	return VINF_SUCCESS;
7245	}
7246
7247
7248	/**
7249	* Leaves the VT-x session.
7250	*
7251	* @returns VBox status code.
7252	* @param pVCpu The cross context virtual CPU structure.
7253	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7254	* out-of-sync. Make sure to update the required fields
7255	* before using them.
7256	*
7257	* @remarks No-long-jmp zone!!!
7258	*/
7259	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7260	{
7261	HM_DISABLE_PREEMPT();
7262	HMVMX_ASSERT_CPU_SAFE();
7263	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7264	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7265
7266	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7267	and done this from the VMXR0ThreadCtxCallback(). */
7268	if (!pVCpu->hm.s.fLeaveDone)
7269	{
7270	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7271	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7272	pVCpu->hm.s.fLeaveDone = true;
7273	}
7274	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7275
7276	/*
7277	* !!! IMPORTANT !!!
7278	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7279	*/
7280
7281	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7282	/** @todo Deregistering here means we need to VMCLEAR always
7283	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7284	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7285	VMMR0ThreadCtxHookDisable(pVCpu);
7286
7287	/* Leave HM context. This takes care of local init (term). */
7288	int rc = HMR0LeaveCpu(pVCpu);
7289
7290	HM_RESTORE_PREEMPT();
7291	return rc;
7292	}
7293
7294
7295	/**
7296	* Does the necessary state syncing before doing a longjmp to ring-3.
7297	*
7298	* @returns VBox status code.
7299	* @param pVCpu The cross context virtual CPU structure.
7300	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7301	* out-of-sync. Make sure to update the required fields
7302	* before using them.
7303	*
7304	* @remarks No-long-jmp zone!!!
7305	*/
7306	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7307	{
7308	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7309	}
7310
7311
7312	/**
7313	* Take necessary actions before going back to ring-3.
7314	*
7315	* An action requires us to go back to ring-3. This function does the necessary
7316	* steps before we can safely return to ring-3. This is not the same as longjmps
7317	* to ring-3, this is voluntary and prepares the guest so it may continue
7318	* executing outside HM (recompiler/IEM).
7319	*
7320	* @returns VBox status code.
7321	* @param pVM The cross context VM structure.
7322	* @param pVCpu The cross context virtual CPU structure.
7323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7324	* out-of-sync. Make sure to update the required fields
7325	* before using them.
7326	* @param rcExit The reason for exiting to ring-3. Can be
7327	* VINF_VMM_UNKNOWN_RING3_CALL.
7328	*/
7329	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7330	{
7331	Assert(pVM);
7332	Assert(pVCpu);
7333	Assert(pMixedCtx);
7334	HMVMX_ASSERT_PREEMPT_SAFE();
7335
7336	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7337	{
7338	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7339	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7340	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7341	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7342	}
7343
7344	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7345	VMMRZCallRing3Disable(pVCpu);
7346	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7347
7348	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7349	if (pVCpu->hm.s.Event.fPending)
7350	{
7351	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7352	Assert(!pVCpu->hm.s.Event.fPending);
7353	}
7354
7355	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7356	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7357
7358	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7359	and if we're injecting an event we should have a TRPM trap pending. */
7360	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7361	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7362	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7363	#endif
7364
7365	/* Save guest state and restore host state bits. */
7366	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7367	AssertRCReturn(rc, rc);
7368	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7369	/* Thread-context hooks are unregistered at this point!!! */
7370
7371	/* Sync recompiler state. */
7372	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7373	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7374	\| CPUM_CHANGED_LDTR
7375	\| CPUM_CHANGED_GDTR
7376	\| CPUM_CHANGED_IDTR
7377	\| CPUM_CHANGED_TR
7378	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7379	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7380	if ( pVM->hm.s.fNestedPaging
7381	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7382	{
7383	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7384	}
7385
7386	Assert(!pVCpu->hm.s.fClearTrapFlag);
7387
7388	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7389	if (rcExit != VINF_EM_RAW_INTERRUPT)
7390	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7391
7392	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7393
7394	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7395	VMMRZCallRing3RemoveNotification(pVCpu);
7396	VMMRZCallRing3Enable(pVCpu);
7397
7398	return rc;
7399	}
7400
7401
7402	/**
7403	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7404	* longjump to ring-3 and possibly get preempted.
7405	*
7406	* @returns VBox status code.
7407	* @param pVCpu The cross context virtual CPU structure.
7408	* @param enmOperation The operation causing the ring-3 longjump.
7409	* @param pvUser Opaque pointer to the guest-CPU context. The data
7410	* may be out-of-sync. Make sure to update the required
7411	* fields before using them.
7412	*/
7413	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7414	{
7415	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7416	{
7417	/*
7418	* !!! IMPORTANT !!!
7419	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7420	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7421	*/
7422	VMMRZCallRing3RemoveNotification(pVCpu);
7423	VMMRZCallRing3Disable(pVCpu);
7424	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7425	RTThreadPreemptDisable(&PreemptState);
7426
7427	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7428	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7429
7430	#if HC_ARCH_BITS == 64
7431	/* Restore host-state bits that VT-x only restores partially. */
7432	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7433	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7434	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7435	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7436	#endif
7437	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7438	if (pVCpu->hm.s.vmx.fLazyMsrs)
7439	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7440
7441	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7442	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7443	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7444	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7445	{
7446	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7447	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7448	}
7449
7450	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7451	VMMR0ThreadCtxHookDisable(pVCpu);
7452	HMR0LeaveCpu(pVCpu);
7453	RTThreadPreemptRestore(&PreemptState);
7454	return VINF_SUCCESS;
7455	}
7456
7457	Assert(pVCpu);
7458	Assert(pvUser);
7459	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7460	HMVMX_ASSERT_PREEMPT_SAFE();
7461
7462	VMMRZCallRing3Disable(pVCpu);
7463	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7464
7465	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7466	enmOperation));
7467
7468	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7469	AssertRCReturn(rc, rc);
7470
7471	VMMRZCallRing3Enable(pVCpu);
7472	return VINF_SUCCESS;
7473	}
7474
7475
7476	/**
7477	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7478	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7479	*
7480	* @param pVCpu The cross context virtual CPU structure.
7481	*/
7482	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7483	{
7484	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7485	{
7486	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7487	{
7488	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7489	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7490	AssertRC(rc);
7491	Log4(("Setup interrupt-window exiting\n"));
7492	}
7493	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7494	}
7495
7496
7497	/**
7498	* Clears the interrupt-window exiting control in the VMCS.
7499	*
7500	* @param pVCpu The cross context virtual CPU structure.
7501	*/
7502	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7503	{
7504	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7505	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7506	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7507	AssertRC(rc);
7508	Log4(("Cleared interrupt-window exiting\n"));
7509	}
7510
7511
7512	/**
7513	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7514	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7515	*
7516	* @param pVCpu The cross context virtual CPU structure.
7517	*/
7518	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7519	{
7520	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7521	{
7522	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7523	{
7524	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7525	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7526	AssertRC(rc);
7527	Log4(("Setup NMI-window exiting\n"));
7528	}
7529	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7530	}
7531
7532
7533	/**
7534	* Clears the NMI-window exiting control in the VMCS.
7535	*
7536	* @param pVCpu The cross context virtual CPU structure.
7537	*/
7538	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7539	{
7540	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7541	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7542	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7543	AssertRC(rc);
7544	Log4(("Cleared NMI-window exiting\n"));
7545	}
7546
7547
7548	/**
7549	* Evaluates the event to be delivered to the guest and sets it as the pending
7550	* event.
7551	*
7552	* @returns The VT-x guest-interruptibility state.
7553	* @param pVCpu The cross context virtual CPU structure.
7554	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7555	* out-of-sync. Make sure to update the required fields
7556	* before using them.
7557	*/
7558	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7559	{
7560	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7561	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7562	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7563	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7564	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7565
7566	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7567	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7568	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7569	Assert(!TRPMHasTrap(pVCpu));
7570
7571	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7572	APICUpdatePendingInterrupts(pVCpu);
7573
7574	/*
7575	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7576	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7577	*/
7578	/** @todo SMI. SMIs take priority over NMIs. */
7579	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7580	{
7581	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7582	if ( !pVCpu->hm.s.Event.fPending
7583	&& !fBlockNmi
7584	&& !fBlockSti
7585	&& !fBlockMovSS)
7586	{
7587	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7588	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7589	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7590
7591	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7592	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7593	}
7594	else
7595	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7596	}
7597	/*
7598	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7599	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7600	*/
7601	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7602	&& !pVCpu->hm.s.fSingleInstruction)
7603	{
7604	Assert(!DBGFIsStepping(pVCpu));
7605	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7606	AssertRC(rc);
7607	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7608	if ( !pVCpu->hm.s.Event.fPending
7609	&& !fBlockInt
7610	&& !fBlockSti
7611	&& !fBlockMovSS)
7612	{
7613	uint8_t u8Interrupt;
7614	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7615	if (RT_SUCCESS(rc))
7616	{
7617	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7618	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7619	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7620
7621	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7622	}
7623	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7624	{
7625	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7626	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7627	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7628
7629	/*
7630	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7631	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7632	* need to re-set this force-flag here.
7633	*/
7634	}
7635	else
7636	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7637	}
7638	else
7639	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7640	}
7641
7642	return uIntrState;
7643	}
7644
7645
7646	/**
7647	* Sets a pending-debug exception to be delivered to the guest if the guest is
7648	* single-stepping in the VMCS.
7649	*
7650	* @param pVCpu The cross context virtual CPU structure.
7651	*/
7652	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7653	{
7654	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7655	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7656	AssertRC(rc);
7657	}
7658
7659
7660	/**
7661	* Injects any pending events into the guest if the guest is in a state to
7662	* receive them.
7663	*
7664	* @returns Strict VBox status code (i.e. informational status codes too).
7665	* @param pVCpu The cross context virtual CPU structure.
7666	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7667	* out-of-sync. Make sure to update the required fields
7668	* before using them.
7669	* @param uIntrState The VT-x guest-interruptibility state.
7670	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7671	* return VINF_EM_DBG_STEPPED if the event was
7672	* dispatched directly.
7673	*/
7674	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7675	{
7676	HMVMX_ASSERT_PREEMPT_SAFE();
7677	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7678
7679	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7680	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7681
7682	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7683	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7684	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7685	Assert(!TRPMHasTrap(pVCpu));
7686
7687	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7688	if (pVCpu->hm.s.Event.fPending)
7689	{
7690	/*
7691	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7692	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7693	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7694	*
7695	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7696	*/
7697	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7698	#ifdef VBOX_STRICT
7699	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7700	{
7701	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7702	Assert(!fBlockInt);
7703	Assert(!fBlockSti);
7704	Assert(!fBlockMovSS);
7705	}
7706	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7707	{
7708	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7709	Assert(!fBlockSti);
7710	Assert(!fBlockMovSS);
7711	Assert(!fBlockNmi);
7712	}
7713	#endif
7714	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7715	(uint8_t)uIntType));
7716	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7717	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7718	fStepping, &uIntrState);
7719	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7720
7721	/* Update the interruptibility-state as it could have been changed by
7722	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7723	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7724	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7725
7726	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7727	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7728	else
7729	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7730	}
7731
7732	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7733	if ( fBlockSti
7734	\|\| fBlockMovSS)
7735	{
7736	if (!pVCpu->hm.s.fSingleInstruction)
7737	{
7738	/*
7739	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7740	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7741	* See Intel spec. 27.3.4 "Saving Non-Register State".
7742	*/
7743	Assert(!DBGFIsStepping(pVCpu));
7744	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7745	AssertRCReturn(rc2, rc2);
7746	if (pMixedCtx->eflags.Bits.u1TF)
7747	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7748	}
7749	else if (pMixedCtx->eflags.Bits.u1TF)
7750	{
7751	/*
7752	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7753	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7754	*/
7755	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7756	uIntrState = 0;
7757	}
7758	}
7759
7760	/*
7761	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7762	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7763	*/
7764	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7765	AssertRC(rc2);
7766
7767	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7768	NOREF(fBlockMovSS); NOREF(fBlockSti);
7769	return rcStrict;
7770	}
7771
7772
7773	/**
7774	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7775	*
7776	* @param pVCpu The cross context virtual CPU structure.
7777	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7778	* out-of-sync. Make sure to update the required fields
7779	* before using them.
7780	*/
7781	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7782	{
7783	NOREF(pMixedCtx);
7784	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7785	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7786	}
7787
7788
7789	/**
7790	* Injects a double-fault (\#DF) exception into the VM.
7791	*
7792	* @returns Strict VBox status code (i.e. informational status codes too).
7793	* @param pVCpu The cross context virtual CPU structure.
7794	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7795	* out-of-sync. Make sure to update the required fields
7796	* before using them.
7797	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7798	* and should return VINF_EM_DBG_STEPPED if the event
7799	* is injected directly (register modified by us, not
7800	* by hardware on VM-entry).
7801	* @param puIntrState Pointer to the current guest interruptibility-state.
7802	* This interruptibility-state will be updated if
7803	* necessary. This cannot not be NULL.
7804	*/
7805	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7806	{
7807	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7808	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7809	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7810	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7811	fStepping, puIntrState);
7812	}
7813
7814
7815	/**
7816	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
7817	*
7818	* @param pVCpu The cross context virtual CPU structure.
7819	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7820	* out-of-sync. Make sure to update the required fields
7821	* before using them.
7822	*/
7823	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7824	{
7825	NOREF(pMixedCtx);
7826	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7827	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7828	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7829	}
7830
7831
7832	/**
7833	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7834	*
7835	* @param pVCpu The cross context virtual CPU structure.
7836	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7837	* out-of-sync. Make sure to update the required fields
7838	* before using them.
7839	* @param cbInstr The value of RIP that is to be pushed on the guest
7840	* stack.
7841	*/
7842	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7843	{
7844	NOREF(pMixedCtx);
7845	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7846	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7847	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7848	}
7849
7850
7851	/**
7852	* Injects a general-protection (\#GP) fault into the VM.
7853	*
7854	* @returns Strict VBox status code (i.e. informational status codes too).
7855	* @param pVCpu The cross context virtual CPU structure.
7856	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7857	* out-of-sync. Make sure to update the required fields
7858	* before using them.
7859	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7860	* mode, i.e. in real-mode it's not valid).
7861	* @param u32ErrorCode The error code associated with the \#GP.
7862	* @param fStepping Whether we're running in
7863	* hmR0VmxRunGuestCodeStep() and should return
7864	* VINF_EM_DBG_STEPPED if the event is injected
7865	* directly (register modified by us, not by
7866	* hardware on VM-entry).
7867	* @param puIntrState Pointer to the current guest interruptibility-state.
7868	* This interruptibility-state will be updated if
7869	* necessary. This cannot not be NULL.
7870	*/
7871	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7872	bool fStepping, uint32_t *puIntrState)
7873	{
7874	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7875	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7876	if (fErrorCodeValid)
7877	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7878	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7879	fStepping, puIntrState);
7880	}
7881
7882
7883	#if 0 /* unused */
7884	/**
7885	* Sets a general-protection (\#GP) exception as pending-for-injection into the
7886	* VM.
7887	*
7888	* @param pVCpu The cross context virtual CPU structure.
7889	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7890	* out-of-sync. Make sure to update the required fields
7891	* before using them.
7892	* @param u32ErrorCode The error code associated with the \#GP.
7893	*/
7894	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7895	{
7896	NOREF(pMixedCtx);
7897	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7898	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7899	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7900	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
7901	}
7902	#endif /* unused */
7903
7904
7905	/**
7906	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7907	*
7908	* @param pVCpu The cross context virtual CPU structure.
7909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7910	* out-of-sync. Make sure to update the required fields
7911	* before using them.
7912	* @param uVector The software interrupt vector number.
7913	* @param cbInstr The value of RIP that is to be pushed on the guest
7914	* stack.
7915	*/
7916	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7917	{
7918	NOREF(pMixedCtx);
7919	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7920	if ( uVector == X86_XCPT_BP
7921	\|\| uVector == X86_XCPT_OF)
7922	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7923	else
7924	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7925	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7926	}
7927
7928
7929	/**
7930	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7931	* stack.
7932	*
7933	* @returns Strict VBox status code (i.e. informational status codes too).
7934	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7935	* @param pVM The cross context VM structure.
7936	* @param pMixedCtx Pointer to the guest-CPU context.
7937	* @param uValue The value to push to the guest stack.
7938	*/
7939	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7940	{
7941	/*
7942	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7943	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7944	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7945	*/
7946	if (pMixedCtx->sp == 1)
7947	return VINF_EM_RESET;
7948	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7949	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7950	AssertRC(rc);
7951	return rc;
7952	}
7953
7954
7955	/**
7956	* Injects an event into the guest upon VM-entry by updating the relevant fields
7957	* in the VM-entry area in the VMCS.
7958	*
7959	* @returns Strict VBox status code (i.e. informational status codes too).
7960	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7961	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7962	*
7963	* @param pVCpu The cross context virtual CPU structure.
7964	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7965	* be out-of-sync. Make sure to update the required
7966	* fields before using them.
7967	* @param u64IntInfo The VM-entry interruption-information field.
7968	* @param cbInstr The VM-entry instruction length in bytes (for
7969	* software interrupts, exceptions and privileged
7970	* software exceptions).
7971	* @param u32ErrCode The VM-entry exception error code.
7972	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7973	* @param puIntrState Pointer to the current guest interruptibility-state.
7974	* This interruptibility-state will be updated if
7975	* necessary. This cannot not be NULL.
7976	* @param fStepping Whether we're running in
7977	* hmR0VmxRunGuestCodeStep() and should return
7978	* VINF_EM_DBG_STEPPED if the event is injected
7979	* directly (register modified by us, not by
7980	* hardware on VM-entry).
7981	*
7982	* @remarks Requires CR0!
7983	* @remarks No-long-jump zone!!!
7984	*/
7985	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7986	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7987	uint32_t *puIntrState)
7988	{
7989	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7990	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7991	Assert(puIntrState);
7992	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7993
7994	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7995	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7996
7997	#ifdef VBOX_STRICT
7998	/* Validate the error-code-valid bit for hardware exceptions. */
7999	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8000	{
8001	switch (uVector)
8002	{
8003	case X86_XCPT_PF:
8004	case X86_XCPT_DF:
8005	case X86_XCPT_TS:
8006	case X86_XCPT_NP:
8007	case X86_XCPT_SS:
8008	case X86_XCPT_GP:
8009	case X86_XCPT_AC:
8010	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8011	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8012	/* fallthru */
8013	default:
8014	break;
8015	}
8016	}
8017	#endif
8018
8019	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8020	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8021	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8022
8023	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8024
8025	/* We require CR0 to check if the guest is in real-mode. */
8026	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8027	AssertRCReturn(rc, rc);
8028
8029	/*
8030	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8031	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8032	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8033	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8034	*/
8035	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8036	{
8037	PVM pVM = pVCpu->CTX_SUFF(pVM);
8038	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8039	{
8040	Assert(PDMVmmDevHeapIsEnabled(pVM));
8041	Assert(pVM->hm.s.vmx.pRealModeTSS);
8042
8043	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8044	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8045	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8046	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8047	AssertRCReturn(rc, rc);
8048	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8049
8050	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8051	size_t const cbIdtEntry = sizeof(X86IDTR16);
8052	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8053	{
8054	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8055	if (uVector == X86_XCPT_DF)
8056	return VINF_EM_RESET;
8057
8058	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8059	if (uVector == X86_XCPT_GP)
8060	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8061
8062	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8063	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8064	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8065	fStepping, puIntrState);
8066	}
8067
8068	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8069	uint16_t uGuestIp = pMixedCtx->ip;
8070	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8071	{
8072	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8073	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8074	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8075	}
8076	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8077	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8078
8079	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8080	X86IDTR16 IdtEntry;
8081	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8082	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8083	AssertRCReturn(rc, rc);
8084
8085	/* Construct the stack frame for the interrupt/exception handler. */
8086	VBOXSTRICTRC rcStrict;
8087	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8088	if (rcStrict == VINF_SUCCESS)
8089	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8090	if (rcStrict == VINF_SUCCESS)
8091	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8092
8093	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8094	if (rcStrict == VINF_SUCCESS)
8095	{
8096	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8097	pMixedCtx->rip = IdtEntry.offSel;
8098	pMixedCtx->cs.Sel = IdtEntry.uSel;
8099	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8100	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8101	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8102	&& uVector == X86_XCPT_PF)
8103	pMixedCtx->cr2 = GCPtrFaultAddress;
8104
8105	/* If any other guest-state bits are changed here, make sure to update
8106	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8107	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8108	\| HM_CHANGED_GUEST_RIP
8109	\| HM_CHANGED_GUEST_RFLAGS
8110	\| HM_CHANGED_GUEST_RSP);
8111
8112	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8113	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8114	{
8115	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8116	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8117	Log4(("Clearing inhibition due to STI.\n"));
8118	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8119	}
8120	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8121	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8122
8123	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8124	it, if we are returning to ring-3 before executing guest code. */
8125	pVCpu->hm.s.Event.fPending = false;
8126
8127	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8128	if (fStepping)
8129	rcStrict = VINF_EM_DBG_STEPPED;
8130	}
8131	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8132	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8133	return rcStrict;
8134	}
8135
8136	/*
8137	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8138	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8139	*/
8140	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8141	}
8142
8143	/* Validate. */
8144	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8145	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8146	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8147
8148	/* Inject. */
8149	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8150	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8151	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8152	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8153
8154	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8155	&& uVector == X86_XCPT_PF)
8156	pMixedCtx->cr2 = GCPtrFaultAddress;
8157
8158	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8159	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8160
8161	AssertRCReturn(rc, rc);
8162	return VINF_SUCCESS;
8163	}
8164
8165
8166	/**
8167	* Clears the interrupt-window exiting control in the VMCS and if necessary
8168	* clears the current event in the VMCS as well.
8169	*
8170	* @returns VBox status code.
8171	* @param pVCpu The cross context virtual CPU structure.
8172	*
8173	* @remarks Use this function only to clear events that have not yet been
8174	* delivered to the guest but are injected in the VMCS!
8175	* @remarks No-long-jump zone!!!
8176	*/
8177	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8178	{
8179	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8180
8181	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8182	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8183
8184	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8185	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8186	}
8187
8188
8189	/**
8190	* Enters the VT-x session.
8191	*
8192	* @returns VBox status code.
8193	* @param pVM The cross context VM structure.
8194	* @param pVCpu The cross context virtual CPU structure.
8195	* @param pCpu Pointer to the CPU info struct.
8196	*/
8197	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8198	{
8199	AssertPtr(pVM);
8200	AssertPtr(pVCpu);
8201	Assert(pVM->hm.s.vmx.fSupported);
8202	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8203	NOREF(pCpu); NOREF(pVM);
8204
8205	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8206	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8207
8208	#ifdef VBOX_STRICT
8209	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8210	RTCCUINTREG uHostCR4 = ASMGetCR4();
8211	if (!(uHostCR4 & X86_CR4_VMXE))
8212	{
8213	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8214	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8215	}
8216	#endif
8217
8218	/*
8219	* Load the VCPU's VMCS as the current (and active) one.
8220	*/
8221	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8222	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8223	if (RT_FAILURE(rc))
8224	return rc;
8225
8226	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8227	pVCpu->hm.s.fLeaveDone = false;
8228	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8229
8230	return VINF_SUCCESS;
8231	}
8232
8233
8234	/**
8235	* The thread-context callback (only on platforms which support it).
8236	*
8237	* @param enmEvent The thread-context event.
8238	* @param pVCpu The cross context virtual CPU structure.
8239	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8240	* @thread EMT(pVCpu)
8241	*/
8242	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8243	{
8244	NOREF(fGlobalInit);
8245
8246	switch (enmEvent)
8247	{
8248	case RTTHREADCTXEVENT_OUT:
8249	{
8250	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8251	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8252	VMCPU_ASSERT_EMT(pVCpu);
8253
8254	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8255
8256	/* No longjmps (logger flushes, locks) in this fragile context. */
8257	VMMRZCallRing3Disable(pVCpu);
8258	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8259
8260	/*
8261	* Restore host-state (FPU, debug etc.)
8262	*/
8263	if (!pVCpu->hm.s.fLeaveDone)
8264	{
8265	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8266	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8267	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8268	pVCpu->hm.s.fLeaveDone = true;
8269	}
8270
8271	/* Leave HM context, takes care of local init (term). */
8272	int rc = HMR0LeaveCpu(pVCpu);
8273	AssertRC(rc); NOREF(rc);
8274
8275	/* Restore longjmp state. */
8276	VMMRZCallRing3Enable(pVCpu);
8277	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8278	break;
8279	}
8280
8281	case RTTHREADCTXEVENT_IN:
8282	{
8283	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8284	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8285	VMCPU_ASSERT_EMT(pVCpu);
8286
8287	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8288	VMMRZCallRing3Disable(pVCpu);
8289	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8290
8291	/* Initialize the bare minimum state required for HM. This takes care of
8292	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8293	int rc = HMR0EnterCpu(pVCpu);
8294	AssertRC(rc);
8295	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8296
8297	/* Load the active VMCS as the current one. */
8298	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8299	{
8300	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8301	AssertRC(rc); NOREF(rc);
8302	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8303	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8304	}
8305	pVCpu->hm.s.fLeaveDone = false;
8306
8307	/* Restore longjmp state. */
8308	VMMRZCallRing3Enable(pVCpu);
8309	break;
8310	}
8311
8312	default:
8313	break;
8314	}
8315	}
8316
8317
8318	/**
8319	* Saves the host state in the VMCS host-state.
8320	* Sets up the VM-exit MSR-load area.
8321	*
8322	* The CPU state will be loaded from these fields on every successful VM-exit.
8323	*
8324	* @returns VBox status code.
8325	* @param pVM The cross context VM structure.
8326	* @param pVCpu The cross context virtual CPU structure.
8327	*
8328	* @remarks No-long-jump zone!!!
8329	*/
8330	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8331	{
8332	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8333
8334	int rc = VINF_SUCCESS;
8335	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8336	{
8337	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8338	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8339
8340	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8341	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8342
8343	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8344	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8345
8346	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8347	}
8348	return rc;
8349	}
8350
8351
8352	/**
8353	* Saves the host state in the VMCS host-state.
8354	*
8355	* @returns VBox status code.
8356	* @param pVM The cross context VM structure.
8357	* @param pVCpu The cross context virtual CPU structure.
8358	*
8359	* @remarks No-long-jump zone!!!
8360	*/
8361	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8362	{
8363	AssertPtr(pVM);
8364	AssertPtr(pVCpu);
8365
8366	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8367
8368	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8369	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8370	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8371	return hmR0VmxSaveHostState(pVM, pVCpu);
8372	}
8373
8374
8375	/**
8376	* Loads the guest state into the VMCS guest-state area.
8377	*
8378	* The will typically be done before VM-entry when the guest-CPU state and the
8379	* VMCS state may potentially be out of sync.
8380	*
8381	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8382	* VM-entry controls.
8383	* Sets up the appropriate VMX non-root function to execute guest code based on
8384	* the guest CPU mode.
8385	*
8386	* @returns VBox strict status code.
8387	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8388	* without unrestricted guest access and the VMMDev is not presently
8389	* mapped (e.g. EFI32).
8390	*
8391	* @param pVM The cross context VM structure.
8392	* @param pVCpu The cross context virtual CPU structure.
8393	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8394	* out-of-sync. Make sure to update the required fields
8395	* before using them.
8396	*
8397	* @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8398	* caller disables then again on successfull return. Confusing.)
8399	*/
8400	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8401	{
8402	AssertPtr(pVM);
8403	AssertPtr(pVCpu);
8404	AssertPtr(pMixedCtx);
8405	HMVMX_ASSERT_PREEMPT_SAFE();
8406
8407	VMMRZCallRing3Disable(pVCpu);
8408	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8409
8410	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8411
8412	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8413
8414	/* Determine real-on-v86 mode. */
8415	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8416	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8417	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8418	{
8419	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8420	}
8421
8422	/*
8423	* Load the guest-state into the VMCS.
8424	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8425	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8426	*/
8427	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8428	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8429
8430	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8431	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8432	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8433
8434	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8435	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8436	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8437
8438	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8439	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8440
8441	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8442	if (rcStrict == VINF_SUCCESS)
8443	{ /* likely */ }
8444	else
8445	{
8446	VMMRZCallRing3Enable(pVCpu);
8447	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8448	return rcStrict;
8449	}
8450
8451	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8452	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8453	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8454
8455	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8456	determine we don't have to swap EFER after all. */
8457	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8458	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8459
8460	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8461	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8462
8463	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8464	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8465
8466	/*
8467	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8468	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8469	*/
8470	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8471	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8472
8473	/* Clear any unused and reserved bits. */
8474	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8475
8476	VMMRZCallRing3Enable(pVCpu);
8477
8478	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8479	return rc;
8480	}
8481
8482
8483	/**
8484	* Loads the state shared between the host and guest into the VMCS.
8485	*
8486	* @param pVM The cross context VM structure.
8487	* @param pVCpu The cross context virtual CPU structure.
8488	* @param pCtx Pointer to the guest-CPU context.
8489	*
8490	* @remarks No-long-jump zone!!!
8491	*/
8492	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8493	{
8494	NOREF(pVM);
8495
8496	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8497	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8498
8499	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8500	{
8501	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8502	AssertRC(rc);
8503	}
8504
8505	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8506	{
8507	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8508	AssertRC(rc);
8509
8510	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8511	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8512	{
8513	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8514	AssertRC(rc);
8515	}
8516	}
8517
8518	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8519	{
8520	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8521	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8522	}
8523
8524	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8525	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8526	{
8527	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8528	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8529	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8530	AssertRC(rc);
8531	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8532	}
8533
8534	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8535	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8536	}
8537
8538
8539	/**
8540	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8541	*
8542	* @returns Strict VBox status code (i.e. informational status codes too).
8543	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8544	* without unrestricted guest access and the VMMDev is not presently
8545	* mapped (e.g. EFI32).
8546	*
8547	* @param pVM The cross context VM structure.
8548	* @param pVCpu The cross context virtual CPU structure.
8549	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8550	* out-of-sync. Make sure to update the required fields
8551	* before using them.
8552	*/
8553	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8554	{
8555	HMVMX_ASSERT_PREEMPT_SAFE();
8556
8557	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8558	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8559	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8560	#endif
8561
8562	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8563	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8564	{
8565	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8566	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8567	{ /* likely */}
8568	else
8569	{
8570	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8571	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8572	}
8573	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8574	}
8575	else if (HMCPU_CF_VALUE(pVCpu))
8576	{
8577	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8578	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8579	{ /* likely */}
8580	else
8581	{
8582	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8583	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8584	return rcStrict;
8585	}
8586	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8587	}
8588
8589	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8590	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8591	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8592	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8593	return rcStrict;
8594	}
8595
8596
8597	/**
8598	* Does the preparations before executing guest code in VT-x.
8599	*
8600	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8601	* recompiler/IEM. We must be cautious what we do here regarding committing
8602	* guest-state information into the VMCS assuming we assuredly execute the
8603	* guest in VT-x mode.
8604	*
8605	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8606	* the common-state (TRPM/forceflags), we must undo those changes so that the
8607	* recompiler/IEM can (and should) use them when it resumes guest execution.
8608	* Otherwise such operations must be done when we can no longer exit to ring-3.
8609	*
8610	* @returns Strict VBox status code (i.e. informational status codes too).
8611	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8612	* have been disabled.
8613	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8614	* double-fault into the guest.
8615	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8616	* dispatched directly.
8617	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8618	*
8619	* @param pVM The cross context VM structure.
8620	* @param pVCpu The cross context virtual CPU structure.
8621	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8622	* out-of-sync. Make sure to update the required fields
8623	* before using them.
8624	* @param pVmxTransient Pointer to the VMX transient structure.
8625	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8626	* us ignore some of the reasons for returning to
8627	* ring-3, and return VINF_EM_DBG_STEPPED if event
8628	* dispatching took place.
8629	*/
8630	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8631	{
8632	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8633
8634	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8635	PGMRZDynMapFlushAutoSet(pVCpu);
8636	#endif
8637
8638	/* Check force flag actions that might require us to go back to ring-3. */
8639	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8640	if (rcStrict == VINF_SUCCESS)
8641	{ /* FFs doesn't get set all the time. */ }
8642	else
8643	return rcStrict;
8644
8645	if (TRPMHasTrap(pVCpu))
8646	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8647	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8648
8649	/*
8650	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8651	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8652	*/
8653	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8654	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8655	{ /* likely */ }
8656	else
8657	{
8658	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8659	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8660	return rcStrict;
8661	}
8662
8663	/*
8664	* Load the guest state bits, we can handle longjmps/getting preempted here.
8665	*
8666	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8667	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8668	* Hence, this needs to be done -after- injection of events.
8669	*/
8670	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8671	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8672	{ /* likely */ }
8673	else
8674	return rcStrict;
8675
8676	/*
8677	* No longjmps to ring-3 from this point on!!!
8678	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8679	* This also disables flushing of the R0-logger instance (if any).
8680	*/
8681	VMMRZCallRing3Disable(pVCpu);
8682
8683	/*
8684	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8685	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8686	*
8687	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8688	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8689	*
8690	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8691	* executing guest code.
8692	*/
8693	pVmxTransient->fEFlags = ASMIntDisableFlags();
8694
8695	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8696	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8697	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8698	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8699	{
8700	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8701	{
8702	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8703	pVCpu->hm.s.Event.fPending = false;
8704
8705	return VINF_SUCCESS;
8706	}
8707
8708	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8709	rcStrict = VINF_EM_RAW_INTERRUPT;
8710	}
8711	else
8712	{
8713	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8714	rcStrict = VINF_EM_RAW_TO_R3;
8715	}
8716
8717	ASMSetFlags(pVmxTransient->fEFlags);
8718	VMMRZCallRing3Enable(pVCpu);
8719
8720	return rcStrict;
8721	}
8722
8723
8724	/**
8725	* Prepares to run guest code in VT-x and we've committed to doing so. This
8726	* means there is no backing out to ring-3 or anywhere else at this
8727	* point.
8728	*
8729	* @param pVM The cross context VM structure.
8730	* @param pVCpu The cross context virtual CPU structure.
8731	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8732	* out-of-sync. Make sure to update the required fields
8733	* before using them.
8734	* @param pVmxTransient Pointer to the VMX transient structure.
8735	*
8736	* @remarks Called with preemption disabled.
8737	* @remarks No-long-jump zone!!!
8738	*/
8739	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8740	{
8741	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8742	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8743	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8744
8745	/*
8746	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8747	*/
8748	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8749	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8750
8751	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8752	if (!CPUMIsGuestFPUStateActive(pVCpu))
8753	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8754	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8755	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8756	#endif
8757
8758	if ( pVCpu->hm.s.fPreloadGuestFpu
8759	&& !CPUMIsGuestFPUStateActive(pVCpu))
8760	{
8761	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8762	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8763	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8764	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8765	}
8766
8767	/*
8768	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8769	*/
8770	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8771	&& pVCpu->hm.s.vmx.cMsrs > 0)
8772	{
8773	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8774	}
8775
8776	/*
8777	* Load the host state bits as we may've been preempted (only happens when
8778	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8779	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8780	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
8781	* See @bugref{8432}.
8782	*/
8783	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8784	{
8785	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8786	AssertRC(rc);
8787	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8788	}
8789	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8790
8791	/*
8792	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8793	*/
8794	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8795	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8796	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8797
8798	/* Store status of the shared guest-host state at the time of VM-entry. */
8799	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8800	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8801	{
8802	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8803	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8804	}
8805	else
8806	#endif
8807	{
8808	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8809	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8810	}
8811	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8812
8813	/*
8814	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8815	*/
8816	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8817	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8818
8819	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8820	RTCPUID idCurrentCpu = pCpu->idCpu;
8821	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8822	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8823	{
8824	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8825	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8826	}
8827
8828	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8829	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8830	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8831	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8832
8833	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8834
8835	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8836	to start executing. */
8837
8838	/*
8839	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8840	*/
8841	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8842	{
8843	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8844	{
8845	bool fMsrUpdated;
8846	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8847	AssertRC(rc2);
8848	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8849
8850	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8851	&fMsrUpdated);
8852	AssertRC(rc2);
8853	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8854
8855	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8856	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8857	}
8858	else
8859	{
8860	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8861	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8862	}
8863	}
8864
8865	#ifdef VBOX_STRICT
8866	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8867	hmR0VmxCheckHostEferMsr(pVCpu);
8868	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8869	#endif
8870	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8871	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8872	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8873	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8874	#endif
8875	}
8876
8877
8878	/**
8879	* Performs some essential restoration of state after running guest code in
8880	* VT-x.
8881	*
8882	* @param pVM The cross context VM structure.
8883	* @param pVCpu The cross context virtual CPU structure.
8884	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8885	* out-of-sync. Make sure to update the required fields
8886	* before using them.
8887	* @param pVmxTransient Pointer to the VMX transient structure.
8888	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8889	*
8890	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
8891	*
8892	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8893	* unconditionally when it is safe to do so.
8894	*/
8895	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8896	{
8897	NOREF(pVM);
8898
8899	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8900
8901	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8902	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8903	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8904	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8905	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8906	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8907
8908	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8909	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8910
8911	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8912	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8913	Assert(!ASMIntAreEnabled());
8914	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8915
8916	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8917	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8918	{
8919	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8920	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8921	}
8922	#endif
8923
8924	#if HC_ARCH_BITS == 64
8925	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8926	#endif
8927	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8928	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8929	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8930	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8931	#else
8932	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8933	#endif
8934	#ifdef VBOX_STRICT
8935	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8936	#endif
8937	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8938	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8939
8940	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8941	uint32_t uExitReason;
8942	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8943	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8944	AssertRC(rc);
8945	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8946	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8947
8948	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8949	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8950	{
8951	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8952	pVmxTransient->fVMEntryFailed));
8953	return;
8954	}
8955
8956	/*
8957	* Update the VM-exit history array here even if the VM-entry failed due to:
8958	* - Invalid guest state.
8959	* - MSR loading.
8960	* - Machine-check event.
8961	*
8962	* In any of the above cases we will still have a "valid" VM-exit reason
8963	* despite @a fVMEntryFailed being false.
8964	*
8965	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8966	*/
8967	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8968
8969	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8970	{
8971	/** @todo We can optimize this by only syncing with our force-flags when
8972	* really needed and keeping the VMCS state as it is for most
8973	* VM-exits. */
8974	/* Update the guest interruptibility-state from the VMCS. */
8975	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8976
8977	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8978	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8979	AssertRC(rc);
8980	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8981	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8982	AssertRC(rc);
8983	#endif
8984
8985	/*
8986	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8987	* we eventually get a VM-exit for any reason.
8988	*
8989	* This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is why it's done here as it's easier and
8990	* no less efficient to deal with it here than making hmR0VmxSaveGuestState() cope with longjmps safely
8991	* (see VMCPU_FF_HM_UPDATE_CR3 handling).
8992	*/
8993	/** @todo r=ramshankar: The 2nd para in the above comment is
8994	* outdated, we no longer longjmp to ring-3 on setting
8995	* the TPR, but regardless we can probably rework this
8996	* portion of the code a bit. */
8997	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8998	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8999	{
9000	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
9001	AssertRC(rc);
9002	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9003	}
9004	}
9005	}
9006
9007
9008	/**
9009	* Runs the guest code using VT-x the normal way.
9010	*
9011	* @returns VBox status code.
9012	* @param pVM The cross context VM structure.
9013	* @param pVCpu The cross context virtual CPU structure.
9014	* @param pCtx Pointer to the guest-CPU context.
9015	*
9016	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9017	*/
9018	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9019	{
9020	VMXTRANSIENT VmxTransient;
9021	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9022	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9023	uint32_t cLoops = 0;
9024
9025	for (;; cLoops++)
9026	{
9027	Assert(!HMR0SuspendPending());
9028	HMVMX_ASSERT_CPU_SAFE();
9029
9030	/* Preparatory work for running guest code, this may force us to return
9031	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9032	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9033	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9034	if (rcStrict != VINF_SUCCESS)
9035	break;
9036
9037	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9038	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9039	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9040
9041	/* Restore any residual host-state and save any bits shared between host
9042	and guest into the guest-CPU state. Re-enables interrupts! */
9043	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9044
9045	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9046	if (RT_SUCCESS(rcRun))
9047	{ /* very likely */ }
9048	else
9049	{
9050	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9051	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9052	return rcRun;
9053	}
9054
9055	/* Profile the VM-exit. */
9056	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9058	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9059	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9060	HMVMX_START_EXIT_DISPATCH_PROF();
9061
9062	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9063
9064	/* Handle the VM-exit. */
9065	#ifdef HMVMX_USE_FUNCTION_TABLE
9066	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9067	#else
9068	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9069	#endif
9070	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9071	if (rcStrict == VINF_SUCCESS)
9072	{
9073	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9074	continue; /* likely */
9075	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9076	rcStrict = VINF_EM_RAW_INTERRUPT;
9077	}
9078	break;
9079	}
9080
9081	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9082	return rcStrict;
9083	}
9084
9085
9086
9087	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9088	* probes.
9089	*
9090	* The following few functions and associated structure contains the bloat
9091	* necessary for providing detailed debug events and dtrace probes as well as
9092	* reliable host side single stepping. This works on the principle of
9093	* "subclassing" the normal execution loop and workers. We replace the loop
9094	* method completely and override selected helpers to add necessary adjustments
9095	* to their core operation.
9096	*
9097	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9098	* any performance for debug and analysis features.
9099	*
9100	* @{
9101	*/
9102
9103	/**
9104	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9105	* the debug run loop.
9106	*/
9107	typedef struct VMXRUNDBGSTATE
9108	{
9109	/** The RIP we started executing at. This is for detecting that we stepped. */
9110	uint64_t uRipStart;
9111	/** The CS we started executing with. */
9112	uint16_t uCsStart;
9113
9114	/** Whether we've actually modified the 1st execution control field. */
9115	bool fModifiedProcCtls : 1;
9116	/** Whether we've actually modified the 2nd execution control field. */
9117	bool fModifiedProcCtls2 : 1;
9118	/** Whether we've actually modified the exception bitmap. */
9119	bool fModifiedXcptBitmap : 1;
9120
9121	/** We desire the modified the CR0 mask to be cleared. */
9122	bool fClearCr0Mask : 1;
9123	/** We desire the modified the CR4 mask to be cleared. */
9124	bool fClearCr4Mask : 1;
9125	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9126	uint32_t fCpe1Extra;
9127	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9128	uint32_t fCpe1Unwanted;
9129	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9130	uint32_t fCpe2Extra;
9131	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9132	uint32_t bmXcptExtra;
9133	/** The sequence number of the Dtrace provider settings the state was
9134	* configured against. */
9135	uint32_t uDtraceSettingsSeqNo;
9136	/** VM-exits to check (one bit per VM-exit). */
9137	uint32_t bmExitsToCheck[3];
9138
9139	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9140	uint32_t fProcCtlsInitial;
9141	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9142	uint32_t fProcCtls2Initial;
9143	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9144	uint32_t bmXcptInitial;
9145	} VMXRUNDBGSTATE;
9146	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9147	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9148
9149
9150	/**
9151	* Initializes the VMXRUNDBGSTATE structure.
9152	*
9153	* @param pVCpu The cross context virtual CPU structure of the
9154	* calling EMT.
9155	* @param pCtx The CPU register context to go with @a pVCpu.
9156	* @param pDbgState The structure to initialize.
9157	*/
9158	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9159	{
9160	pDbgState->uRipStart = pCtx->rip;
9161	pDbgState->uCsStart = pCtx->cs.Sel;
9162
9163	pDbgState->fModifiedProcCtls = false;
9164	pDbgState->fModifiedProcCtls2 = false;
9165	pDbgState->fModifiedXcptBitmap = false;
9166	pDbgState->fClearCr0Mask = false;
9167	pDbgState->fClearCr4Mask = false;
9168	pDbgState->fCpe1Extra = 0;
9169	pDbgState->fCpe1Unwanted = 0;
9170	pDbgState->fCpe2Extra = 0;
9171	pDbgState->bmXcptExtra = 0;
9172	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9173	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9174	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9175	}
9176
9177
9178	/**
9179	* Updates the VMSC fields with changes requested by @a pDbgState.
9180	*
9181	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9182	* immediately before executing guest code, i.e. when interrupts are disabled.
9183	* We don't check status codes here as we cannot easily assert or return in the
9184	* latter case.
9185	*
9186	* @param pVCpu The cross context virtual CPU structure.
9187	* @param pDbgState The debug state.
9188	*/
9189	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9190	{
9191	/*
9192	* Ensure desired flags in VMCS control fields are set.
9193	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9194	*
9195	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9196	* there should be no stale data in pCtx at this point.
9197	*/
9198	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9199	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9200	{
9201	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9202	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9203	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9204	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9205	pDbgState->fModifiedProcCtls = true;
9206	}
9207
9208	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9209	{
9210	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9211	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9212	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9213	pDbgState->fModifiedProcCtls2 = true;
9214	}
9215
9216	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9217	{
9218	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9219	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9220	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9221	pDbgState->fModifiedXcptBitmap = true;
9222	}
9223
9224	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9225	{
9226	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9227	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9228	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9229	}
9230
9231	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9232	{
9233	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9234	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9235	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9236	}
9237	}
9238
9239
9240	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9241	{
9242	/*
9243	* Restore VM-exit control settings as we may not reenter this function the
9244	* next time around.
9245	*/
9246	/* We reload the initial value, trigger what we can of recalculations the
9247	next time around. From the looks of things, that's all that's required atm. */
9248	if (pDbgState->fModifiedProcCtls)
9249	{
9250	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9251	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9252	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9253	AssertRCReturn(rc2, rc2);
9254	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9255	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_DEBUG);
9256	}
9257
9258	/* We're currently the only ones messing with this one, so just restore the
9259	cached value and reload the field. */
9260	if ( pDbgState->fModifiedProcCtls2
9261	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9262	{
9263	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9264	AssertRCReturn(rc2, rc2);
9265	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9266	}
9267
9268	/* If we've modified the exception bitmap, we restore it and trigger
9269	reloading and partial recalculation the next time around. */
9270	if (pDbgState->fModifiedXcptBitmap)
9271	{
9272	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9273	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS \| HM_CHANGED_GUEST_CR0);
9274	}
9275
9276	/* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9277	if (pDbgState->fClearCr0Mask)
9278	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9279
9280	/* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9281	if (pDbgState->fClearCr4Mask)
9282	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9283
9284	return rcStrict;
9285	}
9286
9287
9288	/**
9289	* Configures VM-exit controls for current DBGF and DTrace settings.
9290	*
9291	* This updates @a pDbgState and the VMCS execution control fields to reflect
9292	* the necessary VM-exits demanded by DBGF and DTrace.
9293	*
9294	* @param pVM The cross context VM structure.
9295	* @param pVCpu The cross context virtual CPU structure.
9296	* @param pCtx Pointer to the guest-CPU context.
9297	* @param pDbgState The debug state.
9298	* @param pVmxTransient Pointer to the VMX transient structure. May update
9299	* fUpdateTscOffsettingAndPreemptTimer.
9300	*/
9301	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9302	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9303	{
9304	/*
9305	* Take down the dtrace serial number so we can spot changes.
9306	*/
9307	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9308	ASMCompilerBarrier();
9309
9310	/*
9311	* We'll rebuild most of the middle block of data members (holding the
9312	* current settings) as we go along here, so start by clearing it all.
9313	*/
9314	pDbgState->bmXcptExtra = 0;
9315	pDbgState->fCpe1Extra = 0;
9316	pDbgState->fCpe1Unwanted = 0;
9317	pDbgState->fCpe2Extra = 0;
9318	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9319	pDbgState->bmExitsToCheck[i] = 0;
9320
9321	/*
9322	* Software interrupts (INT XXh) - no idea how to trigger these...
9323	*/
9324	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9325	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9326	{
9327	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9328	}
9329
9330	/*
9331	* INT3 breakpoints - triggered by #BP exceptions.
9332	*/
9333	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9334	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9335
9336	/*
9337	* Exception bitmap and XCPT events+probes.
9338	*/
9339	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9340	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9341	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9342
9343	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9344	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9345	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9346	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9347	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9348	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9349	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9350	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9351	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9352	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9353	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9354	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9355	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9356	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9357	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9358	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9359	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9360	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9361
9362	if (pDbgState->bmXcptExtra)
9363	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9364
9365	/*
9366	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9367	*
9368	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9369	* So, when adding/changing/removing please don't forget to update it.
9370	*
9371	* Some of the macros are picking up local variables to save horizontal space,
9372	* (being able to see it in a table is the lesser evil here).
9373	*/
9374	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9375	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9376	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9377	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9378	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9379	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9380	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9381	} else do { } while (0)
9382	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9383	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9384	{ \
9385	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9386	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9387	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9388	} else do { } while (0)
9389	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9390	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9391	{ \
9392	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9393	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9394	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9395	} else do { } while (0)
9396	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9397	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9398	{ \
9399	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9400	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9401	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9402	} else do { } while (0)
9403
9404	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9405	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9406	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9407	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9408	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9409
9410	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9411	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9412	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9413	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9414	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9415	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9416	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9417	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9418	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9419	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9420	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9421	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9422	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9423	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9424	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9425	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9426	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9427	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9428	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9429	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9430	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9431	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9432	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9433	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9434	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9435	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9436	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9437	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9438	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9439	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9440	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9441	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9442	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9443	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9444	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9445	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9446
9447	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9448	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9449	{
9450	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9451	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9452	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9453	AssertRC(rc2);
9454
9455	#if 0 /** @todo fix me */
9456	pDbgState->fClearCr0Mask = true;
9457	pDbgState->fClearCr4Mask = true;
9458	#endif
9459	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9460	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9461	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9462	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9463	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9464	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9465	require clearing here and in the loop if we start using it. */
9466	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9467	}
9468	else
9469	{
9470	if (pDbgState->fClearCr0Mask)
9471	{
9472	pDbgState->fClearCr0Mask = false;
9473	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9474	}
9475	if (pDbgState->fClearCr4Mask)
9476	{
9477	pDbgState->fClearCr4Mask = false;
9478	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9479	}
9480	}
9481	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9482	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9483
9484	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9485	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9486	{
9487	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9488	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9489	}
9490	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9491	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9492
9493	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9494	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9495	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9496	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9497	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9498	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9499	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9500	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9501	#if 0 /** @todo too slow, fix handler. */
9502	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9503	#endif
9504	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9505
9506	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9507	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9508	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9509	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9510	{
9511	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9512	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9513	}
9514	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9515	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9516	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9517	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9518
9519	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9520	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9521	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9522	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9523	{
9524	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9525	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9526	}
9527	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9528	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9529	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9530	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9531
9532	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9533	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9534	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9535	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9536	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9537	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9538	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9539	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9540	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9541	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9542	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9543	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9544	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9545	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9546	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9547	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9548	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9549	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9550	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9551	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9552	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9553	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9554
9555	#undef IS_EITHER_ENABLED
9556	#undef SET_ONLY_XBM_IF_EITHER_EN
9557	#undef SET_CPE1_XBM_IF_EITHER_EN
9558	#undef SET_CPEU_XBM_IF_EITHER_EN
9559	#undef SET_CPE2_XBM_IF_EITHER_EN
9560
9561	/*
9562	* Sanitize the control stuff.
9563	*/
9564	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9565	if (pDbgState->fCpe2Extra)
9566	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9567	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9568	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9569	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9570	{
9571	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9572	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9573	}
9574
9575	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9576	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9577	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9578	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9579	}
9580
9581
9582	/**
9583	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9584	* appropriate.
9585	*
9586	* The caller has checked the VM-exit against the
9587	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9588	* already, so we don't have to do that either.
9589	*
9590	* @returns Strict VBox status code (i.e. informational status codes too).
9591	* @param pVM The cross context VM structure.
9592	* @param pVCpu The cross context virtual CPU structure.
9593	* @param pMixedCtx Pointer to the guest-CPU context.
9594	* @param pVmxTransient Pointer to the VMX-transient structure.
9595	* @param uExitReason The VM-exit reason.
9596	*
9597	* @remarks The name of this function is displayed by dtrace, so keep it short
9598	* and to the point. No longer than 33 chars long, please.
9599	*/
9600	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9601	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9602	{
9603	/*
9604	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9605	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9606	*
9607	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9608	* does. Must add/change/remove both places. Same ordering, please.
9609	*
9610	* Added/removed events must also be reflected in the next section
9611	* where we dispatch dtrace events.
9612	*/
9613	bool fDtrace1 = false;
9614	bool fDtrace2 = false;
9615	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9616	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9617	uint32_t uEventArg = 0;
9618	#define SET_EXIT(a_EventSubName) \
9619	do { \
9620	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9621	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9622	} while (0)
9623	#define SET_BOTH(a_EventSubName) \
9624	do { \
9625	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9626	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9627	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9628	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9629	} while (0)
9630	switch (uExitReason)
9631	{
9632	case VMX_EXIT_MTF:
9633	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9634
9635	case VMX_EXIT_XCPT_OR_NMI:
9636	{
9637	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9638	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9639	{
9640	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9641	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9642	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9643	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9644	{
9645	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9646	{
9647	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9648	uEventArg = pVmxTransient->uExitIntErrorCode;
9649	}
9650	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9651	switch (enmEvent1)
9652	{
9653	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9654	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9655	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9656	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9657	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9658	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9659	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9660	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9661	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9662	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9663	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9664	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9665	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9666	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9667	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9668	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9669	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9670	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9671	default: break;
9672	}
9673	}
9674	else
9675	AssertFailed();
9676	break;
9677
9678	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9679	uEventArg = idxVector;
9680	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9681	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9682	break;
9683	}
9684	break;
9685	}
9686
9687	case VMX_EXIT_TRIPLE_FAULT:
9688	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9689	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9690	break;
9691	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9692	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9693	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9694	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9695	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9696
9697	/* Instruction specific VM-exits: */
9698	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9699	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9700	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9701	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9702	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9703	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9704	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9705	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9706	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9707	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9708	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9709	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9710	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9711	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9712	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9713	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9714	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9715	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9716	case VMX_EXIT_MOV_CRX:
9717	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9718	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9719	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9720	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9721	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9722	SET_BOTH(CRX_READ);
9723	else
9724	SET_BOTH(CRX_WRITE);
9725	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9726	break;
9727	case VMX_EXIT_MOV_DRX:
9728	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9729	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9730	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9731	SET_BOTH(DRX_READ);
9732	else
9733	SET_BOTH(DRX_WRITE);
9734	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9735	break;
9736	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9737	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9738	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9739	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9740	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9741	case VMX_EXIT_XDTR_ACCESS:
9742	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9743	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9744	{
9745	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9746	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9747	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9748	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9749	}
9750	break;
9751
9752	case VMX_EXIT_TR_ACCESS:
9753	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9754	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9755	{
9756	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9757	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9758	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9759	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9760	}
9761	break;
9762
9763	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9764	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9765	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9766	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9767	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9768	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9769	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9770	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9771	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9772	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9773	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9774
9775	/* Events that aren't relevant at this point. */
9776	case VMX_EXIT_EXT_INT:
9777	case VMX_EXIT_INT_WINDOW:
9778	case VMX_EXIT_NMI_WINDOW:
9779	case VMX_EXIT_TPR_BELOW_THRESHOLD:
9780	case VMX_EXIT_PREEMPT_TIMER:
9781	case VMX_EXIT_IO_INSTR:
9782	break;
9783
9784	/* Errors and unexpected events. */
9785	case VMX_EXIT_INIT_SIGNAL:
9786	case VMX_EXIT_SIPI:
9787	case VMX_EXIT_IO_SMI:
9788	case VMX_EXIT_SMI:
9789	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9790	case VMX_EXIT_ERR_MSR_LOAD:
9791	case VMX_EXIT_ERR_MACHINE_CHECK:
9792	break;
9793
9794	default:
9795	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9796	break;
9797	}
9798	#undef SET_BOTH
9799	#undef SET_EXIT
9800
9801	/*
9802	* Dtrace tracepoints go first. We do them here at once so we don't
9803	* have to copy the guest state saving and stuff a few dozen times.
9804	* Down side is that we've got to repeat the switch, though this time
9805	* we use enmEvent since the probes are a subset of what DBGF does.
9806	*/
9807	if (fDtrace1 \|\| fDtrace2)
9808	{
9809	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9810	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9811	switch (enmEvent1)
9812	{
9813	/** @todo consider which extra parameters would be helpful for each probe. */
9814	case DBGFEVENT_END: break;
9815	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9816	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9817	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9818	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9819	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9820	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9821	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9822	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9823	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9824	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9825	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9826	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9827	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9828	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9829	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9830	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9831	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9832	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9833	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9834	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9835	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9836	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9837	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9838	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9839	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9840	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9841	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9842	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9843	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9844	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9845	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9846	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9847	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9848	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9849	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9850	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9851	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9852	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9853	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9854	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9855	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9856	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9857	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9858	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9859	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9860	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9861	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9862	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9863	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9864	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9865	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9866	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9867	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9868	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9869	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9870	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9871	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9872	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9873	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9874	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9875	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9876	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9877	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9878	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9879	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9880	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9881	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9882	}
9883	switch (enmEvent2)
9884	{
9885	/** @todo consider which extra parameters would be helpful for each probe. */
9886	case DBGFEVENT_END: break;
9887	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9888	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9889	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9890	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9891	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9892	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9893	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9894	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9895	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9896	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9897	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9898	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9899	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9900	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9901	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9902	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9903	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9904	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9905	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9906	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9907	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9908	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9909	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9910	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9911	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9912	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9913	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9914	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9915	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9916	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9917	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9918	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9919	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9920	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9921	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9922	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9923	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9924	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9925	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9926	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9927	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9928	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9929	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9930	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9931	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9932	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9933	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9934	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9935	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9936	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9937	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9938	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9939	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9940	}
9941	}
9942
9943	/*
9944	* Fire of the DBGF event, if enabled (our check here is just a quick one,
9945	* the DBGF call will do a full check).
9946	*
9947	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9948	* Note! If we have to events, we prioritize the first, i.e. the instruction
9949	* one, in order to avoid event nesting.
9950	*/
9951	if ( enmEvent1 != DBGFEVENT_END
9952	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9953	{
9954	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9955	if (rcStrict != VINF_SUCCESS)
9956	return rcStrict;
9957	}
9958	else if ( enmEvent2 != DBGFEVENT_END
9959	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9960	{
9961	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9962	if (rcStrict != VINF_SUCCESS)
9963	return rcStrict;
9964	}
9965
9966	return VINF_SUCCESS;
9967	}
9968
9969
9970	/**
9971	* Single-stepping VM-exit filtering.
9972	*
9973	* This is preprocessing the VM-exits and deciding whether we've gotten far
9974	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9975	* handling is performed.
9976	*
9977	* @returns Strict VBox status code (i.e. informational status codes too).
9978	* @param pVM The cross context VM structure.
9979	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9980	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9981	* out-of-sync. Make sure to update the required
9982	* fields before using them.
9983	* @param pVmxTransient Pointer to the VMX-transient structure.
9984	* @param uExitReason The VM-exit reason.
9985	* @param pDbgState The debug state.
9986	*/
9987	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9988	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9989	{
9990	/*
9991	* Expensive (saves context) generic dtrace VM-exit probe.
9992	*/
9993	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9994	{ /* more likely */ }
9995	else
9996	{
9997	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9998	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9999	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10000	}
10001
10002	/*
10003	* Check for host NMI, just to get that out of the way.
10004	*/
10005	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10006	{ /* normally likely */ }
10007	else
10008	{
10009	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10010	AssertRCReturn(rc2, rc2);
10011	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10012	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10013	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10014	}
10015
10016	/*
10017	* Check for single stepping event if we're stepping.
10018	*/
10019	if (pVCpu->hm.s.fSingleInstruction)
10020	{
10021	switch (uExitReason)
10022	{
10023	case VMX_EXIT_MTF:
10024	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10025
10026	/* Various events: */
10027	case VMX_EXIT_XCPT_OR_NMI:
10028	case VMX_EXIT_EXT_INT:
10029	case VMX_EXIT_TRIPLE_FAULT:
10030	case VMX_EXIT_INT_WINDOW:
10031	case VMX_EXIT_NMI_WINDOW:
10032	case VMX_EXIT_TASK_SWITCH:
10033	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10034	case VMX_EXIT_APIC_ACCESS:
10035	case VMX_EXIT_EPT_VIOLATION:
10036	case VMX_EXIT_EPT_MISCONFIG:
10037	case VMX_EXIT_PREEMPT_TIMER:
10038
10039	/* Instruction specific VM-exits: */
10040	case VMX_EXIT_CPUID:
10041	case VMX_EXIT_GETSEC:
10042	case VMX_EXIT_HLT:
10043	case VMX_EXIT_INVD:
10044	case VMX_EXIT_INVLPG:
10045	case VMX_EXIT_RDPMC:
10046	case VMX_EXIT_RDTSC:
10047	case VMX_EXIT_RSM:
10048	case VMX_EXIT_VMCALL:
10049	case VMX_EXIT_VMCLEAR:
10050	case VMX_EXIT_VMLAUNCH:
10051	case VMX_EXIT_VMPTRLD:
10052	case VMX_EXIT_VMPTRST:
10053	case VMX_EXIT_VMREAD:
10054	case VMX_EXIT_VMRESUME:
10055	case VMX_EXIT_VMWRITE:
10056	case VMX_EXIT_VMXOFF:
10057	case VMX_EXIT_VMXON:
10058	case VMX_EXIT_MOV_CRX:
10059	case VMX_EXIT_MOV_DRX:
10060	case VMX_EXIT_IO_INSTR:
10061	case VMX_EXIT_RDMSR:
10062	case VMX_EXIT_WRMSR:
10063	case VMX_EXIT_MWAIT:
10064	case VMX_EXIT_MONITOR:
10065	case VMX_EXIT_PAUSE:
10066	case VMX_EXIT_XDTR_ACCESS:
10067	case VMX_EXIT_TR_ACCESS:
10068	case VMX_EXIT_INVEPT:
10069	case VMX_EXIT_RDTSCP:
10070	case VMX_EXIT_INVVPID:
10071	case VMX_EXIT_WBINVD:
10072	case VMX_EXIT_XSETBV:
10073	case VMX_EXIT_RDRAND:
10074	case VMX_EXIT_INVPCID:
10075	case VMX_EXIT_VMFUNC:
10076	case VMX_EXIT_RDSEED:
10077	case VMX_EXIT_XSAVES:
10078	case VMX_EXIT_XRSTORS:
10079	{
10080	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10081	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10082	AssertRCReturn(rc2, rc2);
10083	if ( pMixedCtx->rip != pDbgState->uRipStart
10084	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10085	return VINF_EM_DBG_STEPPED;
10086	break;
10087	}
10088
10089	/* Errors and unexpected events: */
10090	case VMX_EXIT_INIT_SIGNAL:
10091	case VMX_EXIT_SIPI:
10092	case VMX_EXIT_IO_SMI:
10093	case VMX_EXIT_SMI:
10094	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10095	case VMX_EXIT_ERR_MSR_LOAD:
10096	case VMX_EXIT_ERR_MACHINE_CHECK:
10097	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10098	break;
10099
10100	default:
10101	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10102	break;
10103	}
10104	}
10105
10106	/*
10107	* Check for debugger event breakpoints and dtrace probes.
10108	*/
10109	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10110	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10111	{
10112	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10113	if (rcStrict != VINF_SUCCESS)
10114	return rcStrict;
10115	}
10116
10117	/*
10118	* Normal processing.
10119	*/
10120	#ifdef HMVMX_USE_FUNCTION_TABLE
10121	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10122	#else
10123	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10124	#endif
10125	}
10126
10127
10128	/**
10129	* Single steps guest code using VT-x.
10130	*
10131	* @returns Strict VBox status code (i.e. informational status codes too).
10132	* @param pVM The cross context VM structure.
10133	* @param pVCpu The cross context virtual CPU structure.
10134	* @param pCtx Pointer to the guest-CPU context.
10135	*
10136	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10137	*/
10138	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10139	{
10140	VMXTRANSIENT VmxTransient;
10141	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10142
10143	/* Set HMCPU indicators. */
10144	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10145	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10146	pVCpu->hm.s.fDebugWantRdTscExit = false;
10147	pVCpu->hm.s.fUsingDebugLoop = true;
10148
10149	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10150	VMXRUNDBGSTATE DbgState;
10151	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10152	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10153
10154	/*
10155	* The loop.
10156	*/
10157	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10158	for (uint32_t cLoops = 0; ; cLoops++)
10159	{
10160	Assert(!HMR0SuspendPending());
10161	HMVMX_ASSERT_CPU_SAFE();
10162	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10163
10164	/*
10165	* Preparatory work for running guest code, this may force us to return
10166	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10167	*/
10168	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10169	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10170	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10171	if (rcStrict != VINF_SUCCESS)
10172	break;
10173
10174	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10175	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10176
10177	/*
10178	* Now we can run the guest code.
10179	*/
10180	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10181
10182	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10183
10184	/*
10185	* Restore any residual host-state and save any bits shared between host
10186	* and guest into the guest-CPU state. Re-enables interrupts!
10187	*/
10188	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10189
10190	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10191	if (RT_SUCCESS(rcRun))
10192	{ /* very likely */ }
10193	else
10194	{
10195	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10196	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10197	return rcRun;
10198	}
10199
10200	/* Profile the VM-exit. */
10201	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10202	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10203	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10204	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10205	HMVMX_START_EXIT_DISPATCH_PROF();
10206
10207	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10208
10209	/*
10210	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10211	*/
10212	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10213	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10214	if (rcStrict != VINF_SUCCESS)
10215	break;
10216	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10217	{
10218	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10219	rcStrict = VINF_EM_RAW_INTERRUPT;
10220	break;
10221	}
10222
10223	/*
10224	* Stepping: Did the RIP change, if so, consider it a single step.
10225	* Otherwise, make sure one of the TFs gets set.
10226	*/
10227	if (fStepping)
10228	{
10229	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10230	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10231	AssertRCReturn(rc2, rc2);
10232	if ( pCtx->rip != DbgState.uRipStart
10233	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10234	{
10235	rcStrict = VINF_EM_DBG_STEPPED;
10236	break;
10237	}
10238	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10239	}
10240
10241	/*
10242	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10243	*/
10244	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10245	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10246	}
10247
10248	/*
10249	* Clear the X86_EFL_TF if necessary.
10250	*/
10251	if (pVCpu->hm.s.fClearTrapFlag)
10252	{
10253	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10254	AssertRCReturn(rc2, rc2);
10255	pVCpu->hm.s.fClearTrapFlag = false;
10256	pCtx->eflags.Bits.u1TF = 0;
10257	}
10258	/** @todo there seems to be issues with the resume flag when the monitor trap
10259	* flag is pending without being used. Seen early in bios init when
10260	* accessing APIC page in protected mode. */
10261
10262	/*
10263	* Restore VM-exit control settings as we may not reenter this function the
10264	* next time around.
10265	*/
10266	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10267
10268	/* Restore HMCPU indicators. */
10269	pVCpu->hm.s.fUsingDebugLoop = false;
10270	pVCpu->hm.s.fDebugWantRdTscExit = false;
10271	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10272
10273	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10274	return rcStrict;
10275	}
10276
10277
10278	/** @} */
10279
10280
10281	/**
10282	* Checks if any expensive dtrace probes are enabled and we should go to the
10283	* debug loop.
10284	*
10285	* @returns true if we should use debug loop, false if not.
10286	*/
10287	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10288	{
10289	/* It's probably faster to OR the raw 32-bit counter variables together.
10290	Since the variables are in an array and the probes are next to one
10291	another (more or less), we have good locality. So, better read
10292	eight-nine cache lines ever time and only have one conditional, than
10293	128+ conditionals, right? */
10294	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10295	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10296	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10297	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10298	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10299	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10300	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10301	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10302	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10303	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10304	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10305	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10306	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10307	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10308	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10309	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10310	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10311	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10312	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10313	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10314	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10315	) != 0
10316	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10317	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10318	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10319	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10320	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10321	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10322	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10323	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10324	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10325	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10326	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10327	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10328	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10329	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10330	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10331	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10332	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10333	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10334	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10335	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10336	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10337	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10338	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10339	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10340	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10341	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10342	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10343	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10344	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10345	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10346	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10347	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10348	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10349	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10350	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10351	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10352	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10353	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10354	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10355	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10356	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10357	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10358	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10359	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10360	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10361	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10362	) != 0
10363	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10364	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10365	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10366	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10367	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10368	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10369	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10370	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10371	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10372	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10373	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10374	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10375	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10376	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10377	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10378	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10379	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10380	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10381	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10382	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10383	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10384	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10385	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10386	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10387	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10388	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10389	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10390	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10391	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10392	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10393	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10394	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10395	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10396	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10397	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10398	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10399	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10400	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10401	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10402	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10403	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10404	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10405	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10406	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10407	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10408	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10409	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10410	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10411	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10412	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10413	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10414	) != 0;
10415	}
10416
10417
10418	/**
10419	* Runs the guest code using VT-x.
10420	*
10421	* @returns Strict VBox status code (i.e. informational status codes too).
10422	* @param pVM The cross context VM structure.
10423	* @param pVCpu The cross context virtual CPU structure.
10424	* @param pCtx Pointer to the guest-CPU context.
10425	*/
10426	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10427	{
10428	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10429	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10430	HMVMX_ASSERT_PREEMPT_SAFE();
10431
10432	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10433
10434	VBOXSTRICTRC rcStrict;
10435	if ( !pVCpu->hm.s.fUseDebugLoop
10436	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10437	&& !DBGFIsStepping(pVCpu)
10438	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10439	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10440	else
10441	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10442
10443	if (rcStrict == VERR_EM_INTERPRETER)
10444	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10445	else if (rcStrict == VINF_EM_RESET)
10446	rcStrict = VINF_EM_TRIPLE_FAULT;
10447
10448	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10449	if (RT_FAILURE(rc2))
10450	{
10451	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10452	rcStrict = rc2;
10453	}
10454	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10455	return rcStrict;
10456	}
10457
10458
10459	#ifndef HMVMX_USE_FUNCTION_TABLE
10460	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10461	{
10462	# ifdef DEBUG_ramshankar
10463	# define RETURN_EXIT_CALL(a_CallExpr) \
10464	do { \
10465	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10466	VBOXSTRICTRC rcStrict = a_CallExpr; \
10467	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10468	return rcStrict; \
10469	} while (0)
10470	# else
10471	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10472	# endif
10473	switch (rcReason)
10474	{
10475	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10476	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10477	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10478	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10479	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10480	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10481	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10482	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10483	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10484	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10485	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10486	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10487	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10488	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10489	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10490	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10491	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10492	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10493	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10494	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10495	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10496	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10497	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10498	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10499	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10500	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10501	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10502	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10503	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10504	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10505	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10506	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10507	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10508	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10509
10510	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10511	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10512	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10513	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10514	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10515	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10516	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10517	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10518	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10519
10520	case VMX_EXIT_VMCLEAR:
10521	case VMX_EXIT_VMLAUNCH:
10522	case VMX_EXIT_VMPTRLD:
10523	case VMX_EXIT_VMPTRST:
10524	case VMX_EXIT_VMREAD:
10525	case VMX_EXIT_VMRESUME:
10526	case VMX_EXIT_VMWRITE:
10527	case VMX_EXIT_VMXOFF:
10528	case VMX_EXIT_VMXON:
10529	case VMX_EXIT_INVEPT:
10530	case VMX_EXIT_INVVPID:
10531	case VMX_EXIT_VMFUNC:
10532	case VMX_EXIT_XSAVES:
10533	case VMX_EXIT_XRSTORS:
10534	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10535	case VMX_EXIT_ENCLS:
10536	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10537	case VMX_EXIT_PML_FULL:
10538	default:
10539	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10540	}
10541	#undef RETURN_EXIT_CALL
10542	}
10543	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10544
10545
10546	#ifdef VBOX_STRICT
10547	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10548	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10549	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10550
10551	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10552	do { \
10553	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10554	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10555	} while (0)
10556
10557	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10558	do { \
10559	AssertPtr(pVCpu); \
10560	AssertPtr(pMixedCtx); \
10561	AssertPtr(pVmxTransient); \
10562	Assert(pVmxTransient->fVMEntryFailed == false); \
10563	Assert(ASMIntAreEnabled()); \
10564	HMVMX_ASSERT_PREEMPT_SAFE(); \
10565	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10566	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10567	HMVMX_ASSERT_PREEMPT_SAFE(); \
10568	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10569	HMVMX_ASSERT_PREEMPT_CPUID(); \
10570	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10571	} while (0)
10572
10573	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10574	do { \
10575	Log4Func(("\n")); \
10576	} while (0)
10577	#else /* nonstrict builds: */
10578	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10579	do { \
10580	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10581	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10582	} while (0)
10583	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10584	#endif
10585
10586
10587	/**
10588	* Advances the guest RIP by the specified number of bytes.
10589	*
10590	* @param pVCpu The cross context virtual CPU structure.
10591	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10592	* out-of-sync. Make sure to update the required fields
10593	* before using them.
10594	* @param cbInstr Number of bytes to advance the RIP by.
10595	*
10596	* @remarks No-long-jump zone!!!
10597	*/
10598	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10599	{
10600	/* Advance the RIP. */
10601	pMixedCtx->rip += cbInstr;
10602	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10603
10604	/* Update interrupt inhibition. */
10605	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10606	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10607	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10608	}
10609
10610
10611	/**
10612	* Advances the guest RIP after reading it from the VMCS.
10613	*
10614	* @returns VBox status code, no informational status codes.
10615	* @param pVCpu The cross context virtual CPU structure.
10616	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10617	* out-of-sync. Make sure to update the required fields
10618	* before using them.
10619	* @param pVmxTransient Pointer to the VMX transient structure.
10620	*
10621	* @remarks No-long-jump zone!!!
10622	*/
10623	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10624	{
10625	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10626	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10627	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10628	AssertRCReturn(rc, rc);
10629
10630	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10631
10632	/*
10633	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10634	* pending debug exception field as it takes care of priority of events.
10635	*
10636	* See Intel spec. 32.2.1 "Debug Exceptions".
10637	*/
10638	if ( !pVCpu->hm.s.fSingleInstruction
10639	&& pMixedCtx->eflags.Bits.u1TF)
10640	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10641
10642	return VINF_SUCCESS;
10643	}
10644
10645
10646	/**
10647	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10648	* and update error record fields accordingly.
10649	*
10650	* @return VMX_IGS_* return codes.
10651	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10652	* wrong with the guest state.
10653	*
10654	* @param pVM The cross context VM structure.
10655	* @param pVCpu The cross context virtual CPU structure.
10656	* @param pCtx Pointer to the guest-CPU state.
10657	*
10658	* @remarks This function assumes our cache of the VMCS controls
10659	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10660	*/
10661	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10662	{
10663	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10664	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10665	uError = (err); \
10666	break; \
10667	} else do { } while (0)
10668
10669	int rc;
10670	uint32_t uError = VMX_IGS_ERROR;
10671	uint32_t u32Val;
10672	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10673
10674	do
10675	{
10676	/*
10677	* CR0.
10678	*/
10679	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10680	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10681	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10682	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10683	if (fUnrestrictedGuest)
10684	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10685
10686	uint32_t u32GuestCR0;
10687	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10688	AssertRCBreak(rc);
10689	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10690	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10691	if ( !fUnrestrictedGuest
10692	&& (u32GuestCR0 & X86_CR0_PG)
10693	&& !(u32GuestCR0 & X86_CR0_PE))
10694	{
10695	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10696	}
10697
10698	/*
10699	* CR4.
10700	*/
10701	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10702	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10703
10704	uint32_t u32GuestCR4;
10705	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10706	AssertRCBreak(rc);
10707	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10708	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10709
10710	/*
10711	* IA32_DEBUGCTL MSR.
10712	*/
10713	uint64_t u64Val;
10714	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10715	AssertRCBreak(rc);
10716	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10717	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10718	{
10719	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10720	}
10721	uint64_t u64DebugCtlMsr = u64Val;
10722
10723	#ifdef VBOX_STRICT
10724	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10725	AssertRCBreak(rc);
10726	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10727	#endif
10728	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10729
10730	/*
10731	* RIP and RFLAGS.
10732	*/
10733	uint32_t u32Eflags;
10734	#if HC_ARCH_BITS == 64
10735	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10736	AssertRCBreak(rc);
10737	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10738	if ( !fLongModeGuest
10739	\|\| !pCtx->cs.Attr.n.u1Long)
10740	{
10741	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10742	}
10743	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10744	* must be identical if the "IA-32e mode guest" VM-entry
10745	* control is 1 and CS.L is 1. No check applies if the
10746	* CPU supports 64 linear-address bits. */
10747
10748	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10749	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10750	AssertRCBreak(rc);
10751	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10752	VMX_IGS_RFLAGS_RESERVED);
10753	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10754	u32Eflags = u64Val;
10755	#else
10756	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10757	AssertRCBreak(rc);
10758	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10759	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10760	#endif
10761
10762	if ( fLongModeGuest
10763	\|\| ( fUnrestrictedGuest
10764	&& !(u32GuestCR0 & X86_CR0_PE)))
10765	{
10766	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10767	}
10768
10769	uint32_t u32EntryInfo;
10770	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10771	AssertRCBreak(rc);
10772	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10773	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10774	{
10775	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10776	}
10777
10778	/*
10779	* 64-bit checks.
10780	*/
10781	#if HC_ARCH_BITS == 64
10782	if (fLongModeGuest)
10783	{
10784	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10785	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10786	}
10787
10788	if ( !fLongModeGuest
10789	&& (u32GuestCR4 & X86_CR4_PCIDE))
10790	{
10791	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10792	}
10793
10794	/** @todo CR3 field must be such that bits 63:52 and bits in the range
10795	* 51:32 beyond the processor's physical-address width are 0. */
10796
10797	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10798	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10799	{
10800	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10801	}
10802
10803	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10804	AssertRCBreak(rc);
10805	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10806
10807	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10808	AssertRCBreak(rc);
10809	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10810	#endif
10811
10812	/*
10813	* PERF_GLOBAL MSR.
10814	*/
10815	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10816	{
10817	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10818	AssertRCBreak(rc);
10819	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10820	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10821	}
10822
10823	/*
10824	* PAT MSR.
10825	*/
10826	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10827	{
10828	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10829	AssertRCBreak(rc);
10830	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10831	for (unsigned i = 0; i < 8; i++)
10832	{
10833	uint8_t u8Val = (u64Val & 0xff);
10834	if ( u8Val != 0 /* UC */
10835	&& u8Val != 1 /* WC */
10836	&& u8Val != 4 /* WT */
10837	&& u8Val != 5 /* WP */
10838	&& u8Val != 6 /* WB */
10839	&& u8Val != 7 /* UC- */)
10840	{
10841	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10842	}
10843	u64Val >>= 8;
10844	}
10845	}
10846
10847	/*
10848	* EFER MSR.
10849	*/
10850	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10851	{
10852	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10853	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10854	AssertRCBreak(rc);
10855	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10856	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10857	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10858	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10859	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10860	HMVMX_CHECK_BREAK( fUnrestrictedGuest
10861	\|\| !(u32GuestCR0 & X86_CR0_PG)
10862	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10863	VMX_IGS_EFER_LMA_LME_MISMATCH);
10864	}
10865
10866	/*
10867	* Segment registers.
10868	*/
10869	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10870	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10871	if (!(u32Eflags & X86_EFL_VM))
10872	{
10873	/* CS */
10874	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10875	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10876	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10877	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10878	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10879	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10880	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10881	/* CS cannot be loaded with NULL in protected mode. */
10882	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10883	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10884	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
10885	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10886	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
10887	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10888	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10889	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10890	else
10891	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10892
10893	/* SS */
10894	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10895	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10896	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10897	if ( !(pCtx->cr0 & X86_CR0_PE)
10898	\|\| pCtx->cs.Attr.n.u4Type == 3)
10899	{
10900	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10901	}
10902	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10903	{
10904	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10905	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10906	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10907	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10908	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10909	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10910	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10911	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10912	}
10913
10914	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10915	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10916	{
10917	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10918	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10919	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10920	\|\| pCtx->ds.Attr.n.u4Type > 11
10921	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10922	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10923	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10924	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10925	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10926	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10927	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10928	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10929	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10930	}
10931	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10932	{
10933	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10934	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10935	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10936	\|\| pCtx->es.Attr.n.u4Type > 11
10937	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10938	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10939	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10940	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10941	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10942	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10943	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10944	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10945	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10946	}
10947	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10948	{
10949	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10950	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10951	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10952	\|\| pCtx->fs.Attr.n.u4Type > 11
10953	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10954	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10955	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10956	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10957	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10958	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10959	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10960	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10961	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10962	}
10963	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10964	{
10965	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10966	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10967	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10968	\|\| pCtx->gs.Attr.n.u4Type > 11
10969	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10970	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10971	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10972	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10973	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10974	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10975	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10976	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10977	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10978	}
10979	/* 64-bit capable CPUs. */
10980	#if HC_ARCH_BITS == 64
10981	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10982	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10983	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10984	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10985	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10986	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
10987	VMX_IGS_LONGMODE_SS_BASE_INVALID);
10988	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
10989	VMX_IGS_LONGMODE_DS_BASE_INVALID);
10990	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
10991	VMX_IGS_LONGMODE_ES_BASE_INVALID);
10992	#endif
10993	}
10994	else
10995	{
10996	/* V86 mode checks. */
10997	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10998	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10999	{
11000	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11001	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11002	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11003	}
11004	else
11005	{
11006	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11007	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11008	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11009	}
11010
11011	/* CS */
11012	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11013	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11014	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11015	/* SS */
11016	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11017	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11018	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11019	/* DS */
11020	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11021	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11022	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11023	/* ES */
11024	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11025	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11026	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11027	/* FS */
11028	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11029	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11030	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11031	/* GS */
11032	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11033	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11034	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11035	/* 64-bit capable CPUs. */
11036	#if HC_ARCH_BITS == 64
11037	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11038	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11039	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11040	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11041	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11042	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11043	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11044	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11045	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11046	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11047	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11048	#endif
11049	}
11050
11051	/*
11052	* TR.
11053	*/
11054	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11055	/* 64-bit capable CPUs. */
11056	#if HC_ARCH_BITS == 64
11057	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11058	#endif
11059	if (fLongModeGuest)
11060	{
11061	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11062	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11063	}
11064	else
11065	{
11066	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11067	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11068	VMX_IGS_TR_ATTR_TYPE_INVALID);
11069	}
11070	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11071	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11072	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11073	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11074	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11075	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11076	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11077	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11078
11079	/*
11080	* GDTR and IDTR.
11081	*/
11082	#if HC_ARCH_BITS == 64
11083	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11084	AssertRCBreak(rc);
11085	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11086
11087	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11088	AssertRCBreak(rc);
11089	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11090	#endif
11091
11092	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11093	AssertRCBreak(rc);
11094	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11095
11096	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11097	AssertRCBreak(rc);
11098	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11099
11100	/*
11101	* Guest Non-Register State.
11102	*/
11103	/* Activity State. */
11104	uint32_t u32ActivityState;
11105	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11106	AssertRCBreak(rc);
11107	HMVMX_CHECK_BREAK( !u32ActivityState
11108	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11109	VMX_IGS_ACTIVITY_STATE_INVALID);
11110	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11111	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11112	uint32_t u32IntrState;
11113	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11114	AssertRCBreak(rc);
11115	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11116	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11117	{
11118	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11119	}
11120
11121	/** @todo Activity state and injecting interrupts. Left as a todo since we
11122	* currently don't use activity states but ACTIVE. */
11123
11124	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11125	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11126
11127	/* Guest interruptibility-state. */
11128	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11129	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11130	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11131	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11132	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11133	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11134	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11135	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11136	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11137	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11138	{
11139	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11140	{
11141	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11142	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11143	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11144	}
11145	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11146	{
11147	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11148	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11149	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11150	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11151	}
11152	}
11153	/** @todo Assumes the processor is not in SMM. */
11154	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11155	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11156	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11157	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11158	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11159	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11160	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11161	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11162	{
11163	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11164	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11165	}
11166
11167	/* Pending debug exceptions. */
11168	#if HC_ARCH_BITS == 64
11169	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11170	AssertRCBreak(rc);
11171	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11172	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11173	u32Val = u64Val; /* For pending debug exceptions checks below. */
11174	#else
11175	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11176	AssertRCBreak(rc);
11177	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11178	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11179	#endif
11180
11181	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11182	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11183	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11184	{
11185	if ( (u32Eflags & X86_EFL_TF)
11186	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11187	{
11188	/* Bit 14 is PendingDebug.BS. */
11189	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11190	}
11191	if ( !(u32Eflags & X86_EFL_TF)
11192	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11193	{
11194	/* Bit 14 is PendingDebug.BS. */
11195	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11196	}
11197	}
11198
11199	/* VMCS link pointer. */
11200	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11201	AssertRCBreak(rc);
11202	if (u64Val != UINT64_C(0xffffffffffffffff))
11203	{
11204	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11205	/** @todo Bits beyond the processor's physical-address width MBZ. */
11206	/** @todo 32-bit located in memory referenced by value of this field (as a
11207	* physical address) must contain the processor's VMCS revision ID. */
11208	/** @todo SMM checks. */
11209	}
11210
11211	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11212	* not using Nested Paging? */
11213	if ( pVM->hm.s.fNestedPaging
11214	&& !fLongModeGuest
11215	&& CPUMIsGuestInPAEModeEx(pCtx))
11216	{
11217	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11218	AssertRCBreak(rc);
11219	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11220
11221	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11222	AssertRCBreak(rc);
11223	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11224
11225	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11226	AssertRCBreak(rc);
11227	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11228
11229	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11230	AssertRCBreak(rc);
11231	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11232	}
11233
11234	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11235	if (uError == VMX_IGS_ERROR)
11236	uError = VMX_IGS_REASON_NOT_FOUND;
11237	} while (0);
11238
11239	pVCpu->hm.s.u32HMError = uError;
11240	return uError;
11241
11242	#undef HMVMX_ERROR_BREAK
11243	#undef HMVMX_CHECK_BREAK
11244	}
11245
11246	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11247	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11248	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11249
11250	/** @name VM-exit handlers.
11251	* @{
11252	*/
11253
11254	/**
11255	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11256	*/
11257	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11258	{
11259	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11261	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11262	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11263	return VINF_SUCCESS;
11264	return VINF_EM_RAW_INTERRUPT;
11265	}
11266
11267
11268	/**
11269	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11270	*/
11271	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11272	{
11273	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11274	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11275
11276	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11277	AssertRCReturn(rc, rc);
11278
11279	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11280	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11281	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11282	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11283
11284	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11285	{
11286	/*
11287	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11288	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11289	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11290	*
11291	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11292	*/
11293	VMXDispatchHostNmi();
11294	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11295	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11296	return VINF_SUCCESS;
11297	}
11298
11299	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11300	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11301	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11302	{ /* likely */ }
11303	else
11304	{
11305	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11306	rcStrictRc1 = VINF_SUCCESS;
11307	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11308	return rcStrictRc1;
11309	}
11310
11311	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11312	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11313	switch (uIntType)
11314	{
11315	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11316	Assert(uVector == X86_XCPT_DB);
11317	/* no break */
11318	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11319	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11320	/* no break */
11321	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11322	{
11323	/*
11324	* If there's any exception caused as a result of event injection, go back to
11325	* the interpreter. The page-fault case is complicated and we manually handle
11326	* any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11327	* handled in hmR0VmxCheckExitDueToEventDelivery.
11328	*/
11329	if (!pVCpu->hm.s.Event.fPending)
11330	{ /* likely */ }
11331	else if ( uVector != X86_XCPT_PF
11332	&& uVector != X86_XCPT_AC)
11333	{
11334	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11335	rc = VERR_EM_INTERPRETER;
11336	break;
11337	}
11338
11339	switch (uVector)
11340	{
11341	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11342	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11343	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11344	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11345	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11346	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11347	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11348
11349	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11350	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11351	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11352	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11353	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11354	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11355	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11356	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11357	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11358	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11359	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11360	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11361	default:
11362	{
11363	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11364	AssertRCReturn(rc, rc);
11365
11366	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11367	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11368	{
11369	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11370	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11371	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11372
11373	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11374	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11375	AssertRCReturn(rc, rc);
11376	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11377	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11378	0 /* GCPtrFaultAddress */);
11379	AssertRCReturn(rc, rc);
11380	}
11381	else
11382	{
11383	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11384	pVCpu->hm.s.u32HMError = uVector;
11385	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11386	}
11387	break;
11388	}
11389	}
11390	break;
11391	}
11392
11393	default:
11394	{
11395	pVCpu->hm.s.u32HMError = uExitIntInfo;
11396	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11397	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11398	break;
11399	}
11400	}
11401	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11402	return rc;
11403	}
11404
11405
11406	/**
11407	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11408	*/
11409	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11410	{
11411	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11412
11413	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11414	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11415
11416	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11418	return VINF_SUCCESS;
11419	}
11420
11421
11422	/**
11423	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11424	*/
11425	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11426	{
11427	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11428	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11429	{
11430	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11431	HMVMX_RETURN_UNEXPECTED_EXIT();
11432	}
11433
11434	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11435
11436	/*
11437	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11438	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11439	*/
11440	uint32_t uIntrState = 0;
11441	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11442	AssertRCReturn(rc, rc);
11443
11444	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11445	if ( fBlockSti
11446	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11447	{
11448	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11449	}
11450
11451	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11452	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11453
11454	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11455	return VINF_SUCCESS;
11456	}
11457
11458
11459	/**
11460	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11461	*/
11462	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11463	{
11464	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11465	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11466	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11467	}
11468
11469
11470	/**
11471	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11472	*/
11473	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11474	{
11475	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11477	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11478	}
11479
11480
11481	/**
11482	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11483	*/
11484	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11485	{
11486	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11487	PVM pVM = pVCpu->CTX_SUFF(pVM);
11488	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11489	if (RT_LIKELY(rc == VINF_SUCCESS))
11490	{
11491	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11492	Assert(pVmxTransient->cbInstr == 2);
11493	}
11494	else
11495	{
11496	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11497	rc = VERR_EM_INTERPRETER;
11498	}
11499	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11500	return rc;
11501	}
11502
11503
11504	/**
11505	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11506	*/
11507	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11508	{
11509	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11510	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11511	AssertRCReturn(rc, rc);
11512
11513	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11514	return VINF_EM_RAW_EMULATE_INSTR;
11515
11516	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11517	HMVMX_RETURN_UNEXPECTED_EXIT();
11518	}
11519
11520
11521	/**
11522	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11523	*/
11524	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11525	{
11526	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11527	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11528	AssertRCReturn(rc, rc);
11529
11530	PVM pVM = pVCpu->CTX_SUFF(pVM);
11531	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11532	if (RT_LIKELY(rc == VINF_SUCCESS))
11533	{
11534	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11535	Assert(pVmxTransient->cbInstr == 2);
11536	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11537	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11538	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11539	}
11540	else
11541	rc = VERR_EM_INTERPRETER;
11542	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11543	return rc;
11544	}
11545
11546
11547	/**
11548	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11549	*/
11550	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11551	{
11552	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11553	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11554	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11555	AssertRCReturn(rc, rc);
11556
11557	PVM pVM = pVCpu->CTX_SUFF(pVM);
11558	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11559	if (RT_SUCCESS(rc))
11560	{
11561	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11562	Assert(pVmxTransient->cbInstr == 3);
11563	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11564	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11565	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11566	}
11567	else
11568	{
11569	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11570	rc = VERR_EM_INTERPRETER;
11571	}
11572	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11573	return rc;
11574	}
11575
11576
11577	/**
11578	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11579	*/
11580	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11581	{
11582	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11583	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11584	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11585	AssertRCReturn(rc, rc);
11586
11587	PVM pVM = pVCpu->CTX_SUFF(pVM);
11588	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11589	if (RT_LIKELY(rc == VINF_SUCCESS))
11590	{
11591	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11592	Assert(pVmxTransient->cbInstr == 2);
11593	}
11594	else
11595	{
11596	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11597	rc = VERR_EM_INTERPRETER;
11598	}
11599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11600	return rc;
11601	}
11602
11603
11604	/**
11605	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11606	*/
11607	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11608	{
11609	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11611
11612	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11613	if (pVCpu->hm.s.fHypercallsEnabled)
11614	{
11615	#if 0
11616	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11617	#else
11618	/* Aggressive state sync. for now. */
11619	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11620	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11621	AssertRCReturn(rc, rc);
11622	#endif
11623
11624	/* Perform the hypercall. */
11625	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11626	if (rcStrict == VINF_SUCCESS)
11627	{
11628	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11629	AssertRCReturn(rc, rc);
11630	}
11631	else
11632	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11633	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11634	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11635
11636	/* If the hypercall changes anything other than guest's general-purpose registers,
11637	we would need to reload the guest changed bits here before VM-entry. */
11638	}
11639	else
11640	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11641
11642	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11643	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11644	{
11645	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11646	rcStrict = VINF_SUCCESS;
11647	}
11648
11649	return rcStrict;
11650	}
11651
11652
11653	/**
11654	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11655	*/
11656	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11657	{
11658	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11659	PVM pVM = pVCpu->CTX_SUFF(pVM);
11660	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11661
11662	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11663	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11664	AssertRCReturn(rc, rc);
11665
11666	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11667	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11668	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11669	else
11670	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11671	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11672	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11673	return rcStrict;
11674	}
11675
11676
11677	/**
11678	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11679	*/
11680	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11681	{
11682	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11683	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11684	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11685	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11686	AssertRCReturn(rc, rc);
11687
11688	PVM pVM = pVCpu->CTX_SUFF(pVM);
11689	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11690	if (RT_LIKELY(rc == VINF_SUCCESS))
11691	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11692	else
11693	{
11694	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11695	rc = VERR_EM_INTERPRETER;
11696	}
11697	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11698	return rc;
11699	}
11700
11701
11702	/**
11703	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11704	*/
11705	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11706	{
11707	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11708	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11709	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11710	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11711	AssertRCReturn(rc, rc);
11712
11713	PVM pVM = pVCpu->CTX_SUFF(pVM);
11714	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11715	rc = VBOXSTRICTRC_VAL(rc2);
11716	if (RT_LIKELY( rc == VINF_SUCCESS
11717	\|\| rc == VINF_EM_HALT))
11718	{
11719	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11720	AssertRCReturn(rc3, rc3);
11721
11722	if ( rc == VINF_EM_HALT
11723	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11724	{
11725	rc = VINF_SUCCESS;
11726	}
11727	}
11728	else
11729	{
11730	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11731	rc = VERR_EM_INTERPRETER;
11732	}
11733	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11734	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11735	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11736	return rc;
11737	}
11738
11739
11740	/**
11741	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11742	*/
11743	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11744	{
11745	/*
11746	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11747	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11748	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
11749	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11750	*/
11751	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11752	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11753	HMVMX_RETURN_UNEXPECTED_EXIT();
11754	}
11755
11756
11757	/**
11758	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11759	*/
11760	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11761	{
11762	/*
11763	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11764	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11765	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
11766	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11767	*/
11768	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11769	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11770	HMVMX_RETURN_UNEXPECTED_EXIT();
11771	}
11772
11773
11774	/**
11775	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11776	*/
11777	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11778	{
11779	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11780	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11781	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11782	HMVMX_RETURN_UNEXPECTED_EXIT();
11783	}
11784
11785
11786	/**
11787	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11788	*/
11789	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11790	{
11791	/*
11792	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11793	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11794	* See Intel spec. 25.3 "Other Causes of VM-exits".
11795	*/
11796	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11797	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11798	HMVMX_RETURN_UNEXPECTED_EXIT();
11799	}
11800
11801
11802	/**
11803	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11804	* VM-exit.
11805	*/
11806	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11807	{
11808	/*
11809	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11810	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11811	*
11812	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11813	* See Intel spec. "23.8 Restrictions on VMX operation".
11814	*/
11815	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11816	return VINF_SUCCESS;
11817	}
11818
11819
11820	/**
11821	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11822	* VM-exit.
11823	*/
11824	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11825	{
11826	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11827	return VINF_EM_RESET;
11828	}
11829
11830
11831	/**
11832	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11833	*/
11834	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11835	{
11836	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11837	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11838
11839	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11840	AssertRCReturn(rc, rc);
11841
11842	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11843	rc = VINF_SUCCESS;
11844	else
11845	rc = VINF_EM_HALT;
11846
11847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11848	if (rc != VINF_SUCCESS)
11849	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11850	return rc;
11851	}
11852
11853
11854	/**
11855	* VM-exit handler for instructions that result in a \#UD exception delivered to
11856	* the guest.
11857	*/
11858	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11859	{
11860	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11861	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11862	return VINF_SUCCESS;
11863	}
11864
11865
11866	/**
11867	* VM-exit handler for expiry of the VMX preemption timer.
11868	*/
11869	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11870	{
11871	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11872
11873	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11874	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11875
11876	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11877	PVM pVM = pVCpu->CTX_SUFF(pVM);
11878	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11879	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11880	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11881	}
11882
11883
11884	/**
11885	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11886	*/
11887	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11888	{
11889	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11890
11891	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11892	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
11893	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11894	AssertRCReturn(rc, rc);
11895
11896	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11897	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11898
11899	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11900
11901	return rcStrict;
11902	}
11903
11904
11905	/**
11906	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11907	*/
11908	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11909	{
11910	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11911
11912	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11913	/** @todo implement EMInterpretInvpcid() */
11914	return VERR_EM_INTERPRETER;
11915	}
11916
11917
11918	/**
11919	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11920	* Error VM-exit.
11921	*/
11922	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11923	{
11924	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11925	AssertRCReturn(rc, rc);
11926
11927	rc = hmR0VmxCheckVmcsCtls(pVCpu);
11928	AssertRCReturn(rc, rc);
11929
11930	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11931	NOREF(uInvalidReason);
11932
11933	#ifdef VBOX_STRICT
11934	uint32_t uIntrState;
11935	RTHCUINTREG uHCReg;
11936	uint64_t u64Val;
11937	uint32_t u32Val;
11938
11939	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11940	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11941	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11942	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11943	AssertRCReturn(rc, rc);
11944
11945	Log4(("uInvalidReason %u\n", uInvalidReason));
11946	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11947	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11948	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11949	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11950
11951	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11952	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11953	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11954	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11955	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11956	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11957	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11958	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11959	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11960	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11961	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11962	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11963	#else
11964	NOREF(pVmxTransient);
11965	#endif
11966
11967	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11968	return VERR_VMX_INVALID_GUEST_STATE;
11969	}
11970
11971
11972	/**
11973	* VM-exit handler for VM-entry failure due to an MSR-load
11974	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11975	*/
11976	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11977	{
11978	NOREF(pVmxTransient);
11979	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11980	HMVMX_RETURN_UNEXPECTED_EXIT();
11981	}
11982
11983
11984	/**
11985	* VM-exit handler for VM-entry failure due to a machine-check event
11986	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11987	*/
11988	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11989	{
11990	NOREF(pVmxTransient);
11991	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11992	HMVMX_RETURN_UNEXPECTED_EXIT();
11993	}
11994
11995
11996	/**
11997	* VM-exit handler for all undefined reasons. Should never ever happen.. in
11998	* theory.
11999	*/
12000	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12001	{
12002	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12003	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12004	return VERR_VMX_UNDEFINED_EXIT_CODE;
12005	}
12006
12007
12008	/**
12009	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12010	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12011	* Conditional VM-exit.
12012	*/
12013	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12014	{
12015	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12016
12017	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12019	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12020	return VERR_EM_INTERPRETER;
12021	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12022	HMVMX_RETURN_UNEXPECTED_EXIT();
12023	}
12024
12025
12026	/**
12027	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12028	*/
12029	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12030	{
12031	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12032
12033	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12034	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12035	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12036	return VERR_EM_INTERPRETER;
12037	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12038	HMVMX_RETURN_UNEXPECTED_EXIT();
12039	}
12040
12041
12042	/**
12043	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12044	*/
12045	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12046	{
12047	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12048
12049	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12050	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12051	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12052	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12053	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12054	{
12055	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12056	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12057	}
12058	AssertRCReturn(rc, rc);
12059	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12060
12061	#ifdef VBOX_STRICT
12062	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12063	{
12064	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12065	&& pMixedCtx->ecx != MSR_K6_EFER)
12066	{
12067	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12068	pMixedCtx->ecx));
12069	HMVMX_RETURN_UNEXPECTED_EXIT();
12070	}
12071	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12072	{
12073	VMXMSREXITREAD enmRead;
12074	VMXMSREXITWRITE enmWrite;
12075	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12076	AssertRCReturn(rc2, rc2);
12077	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12078	{
12079	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12080	HMVMX_RETURN_UNEXPECTED_EXIT();
12081	}
12082	}
12083	}
12084	#endif
12085
12086	PVM pVM = pVCpu->CTX_SUFF(pVM);
12087	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12088	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12089	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12091	if (RT_SUCCESS(rc))
12092	{
12093	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12094	Assert(pVmxTransient->cbInstr == 2);
12095	}
12096	return rc;
12097	}
12098
12099
12100	/**
12101	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12102	*/
12103	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12104	{
12105	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12106	PVM pVM = pVCpu->CTX_SUFF(pVM);
12107	int rc = VINF_SUCCESS;
12108
12109	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12110	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12111	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12112	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12113	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12114	{
12115	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12116	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12117	}
12118	AssertRCReturn(rc, rc);
12119	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12120
12121	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12122	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12124
12125	if (RT_SUCCESS(rc))
12126	{
12127	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12128
12129	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12130	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12131	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12132	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12133	{
12134	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12135	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12136	EMInterpretWrmsr() changes it. */
12137	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12138	}
12139	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12140	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12141	else if (pMixedCtx->ecx == MSR_K6_EFER)
12142	{
12143	/*
12144	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12145	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12146	* the other bits as well, SCE and NXE. See @bugref{7368}.
12147	*/
12148	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12149	}
12150
12151	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12152	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12153	{
12154	switch (pMixedCtx->ecx)
12155	{
12156	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12157	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12158	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12159	case MSR_K8_FS_BASE: /* no break */
12160	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12161	case MSR_K6_EFER: /* already handled above */ break;
12162	default:
12163	{
12164	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12165	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12166	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12167	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12168	break;
12169	}
12170	}
12171	}
12172	#ifdef VBOX_STRICT
12173	else
12174	{
12175	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12176	switch (pMixedCtx->ecx)
12177	{
12178	case MSR_IA32_SYSENTER_CS:
12179	case MSR_IA32_SYSENTER_EIP:
12180	case MSR_IA32_SYSENTER_ESP:
12181	case MSR_K8_FS_BASE:
12182	case MSR_K8_GS_BASE:
12183	{
12184	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12185	HMVMX_RETURN_UNEXPECTED_EXIT();
12186	}
12187
12188	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12189	default:
12190	{
12191	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12192	{
12193	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12194	if (pMixedCtx->ecx != MSR_K6_EFER)
12195	{
12196	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12197	pMixedCtx->ecx));
12198	HMVMX_RETURN_UNEXPECTED_EXIT();
12199	}
12200	}
12201
12202	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12203	{
12204	VMXMSREXITREAD enmRead;
12205	VMXMSREXITWRITE enmWrite;
12206	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12207	AssertRCReturn(rc2, rc2);
12208	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12209	{
12210	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12211	HMVMX_RETURN_UNEXPECTED_EXIT();
12212	}
12213	}
12214	break;
12215	}
12216	}
12217	}
12218	#endif /* VBOX_STRICT */
12219	}
12220	return rc;
12221	}
12222
12223
12224	/**
12225	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12226	*/
12227	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12228	{
12229	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12230
12231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12232	return VINF_EM_RAW_INTERRUPT;
12233	}
12234
12235
12236	/**
12237	* VM-exit handler for when the TPR value is lowered below the specified
12238	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12239	*/
12240	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12241	{
12242	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12243	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12244
12245	/*
12246	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12247	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12248	* resume guest execution.
12249	*/
12250	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12251	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12252	return VINF_SUCCESS;
12253	}
12254
12255
12256	/**
12257	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12258	* VM-exit.
12259	*
12260	* @retval VINF_SUCCESS when guest execution can continue.
12261	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12262	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12263	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12264	* interpreter.
12265	*/
12266	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12267	{
12268	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12269	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12270	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12271	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12272	AssertRCReturn(rc, rc);
12273
12274	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12275	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12276	PVM pVM = pVCpu->CTX_SUFF(pVM);
12277	VBOXSTRICTRC rcStrict;
12278	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12279	switch (uAccessType)
12280	{
12281	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12282	{
12283	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12284	AssertRCReturn(rc, rc);
12285
12286	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12287	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12288	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12289	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12290	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12291	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12292	{
12293	case 0: /* CR0 */
12294	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12295	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12296	break;
12297	case 2: /* CR2 */
12298	/* Nothing to do here, CR2 it's not part of the VMCS. */
12299	break;
12300	case 3: /* CR3 */
12301	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12302	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12303	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12304	break;
12305	case 4: /* CR4 */
12306	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12307	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12308	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12309	break;
12310	case 8: /* CR8 */
12311	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12312	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12313	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12314	break;
12315	default:
12316	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12317	break;
12318	}
12319
12320	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12321	break;
12322	}
12323
12324	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12325	{
12326	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12327	AssertRCReturn(rc, rc);
12328
12329	Assert( !pVM->hm.s.fNestedPaging
12330	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12331	\|\| pVCpu->hm.s.fUsingDebugLoop
12332	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12333
12334	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12335	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12336	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12337
12338	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12339	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12340	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12341	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12342	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12343	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12344	VBOXSTRICTRC_VAL(rcStrict)));
12345	break;
12346	}
12347
12348	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12349	{
12350	AssertRCReturn(rc, rc);
12351	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12352	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12353	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12354	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12355	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12356	break;
12357	}
12358
12359	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12360	{
12361	AssertRCReturn(rc, rc);
12362	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12363	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12364	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12365	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12366	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12367	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12368	break;
12369	}
12370
12371	default:
12372	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12373	VERR_VMX_UNEXPECTED_EXCEPTION);
12374	}
12375
12376	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12377	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12378	NOREF(pVM);
12379	return rcStrict;
12380	}
12381
12382
12383	/**
12384	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12385	* VM-exit.
12386	*/
12387	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12388	{
12389	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12390	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12391
12392	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12393	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12394	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12395	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12396	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12397	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12398	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12399	AssertRCReturn(rc2, rc2);
12400
12401	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12402	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12403	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12404	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12405	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12406	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12407	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12408	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12409	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12410
12411	/* I/O operation lookup arrays. */
12412	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12413	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12414
12415	VBOXSTRICTRC rcStrict;
12416	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12417	uint32_t const cbInstr = pVmxTransient->cbInstr;
12418	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12419	PVM pVM = pVCpu->CTX_SUFF(pVM);
12420	if (fIOString)
12421	{
12422	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12423	See @bugref{5752#c158}. Should work now. */
12424	/*
12425	* INS/OUTS - I/O String instruction.
12426	*
12427	* Use instruction-information if available, otherwise fall back on
12428	* interpreting the instruction.
12429	*/
12430	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12431	fIOWrite ? 'w' : 'r'));
12432	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12433	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12434	{
12435	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12436	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12437	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12438	AssertRCReturn(rc2, rc2);
12439	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12440	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12441	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12442	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12443	if (fIOWrite)
12444	{
12445	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12446	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12447	}
12448	else
12449	{
12450	/*
12451	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12452	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12453	* See Intel Instruction spec. for "INS".
12454	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12455	*/
12456	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12457	}
12458	}
12459	else
12460	{
12461	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12462	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12463	AssertRCReturn(rc2, rc2);
12464	rcStrict = IEMExecOne(pVCpu);
12465	}
12466	/** @todo IEM needs to be setting these flags somehow. */
12467	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12468	fUpdateRipAlready = true;
12469	#else
12470	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12471	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12472	if (RT_SUCCESS(rcStrict))
12473	{
12474	if (fIOWrite)
12475	{
12476	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12477	(DISCPUMODE)pDis->uAddrMode, cbValue);
12478	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12479	}
12480	else
12481	{
12482	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12483	(DISCPUMODE)pDis->uAddrMode, cbValue);
12484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12485	}
12486	}
12487	else
12488	{
12489	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12490	pMixedCtx->rip));
12491	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12492	}
12493	#endif
12494	}
12495	else
12496	{
12497	/*
12498	* IN/OUT - I/O instruction.
12499	*/
12500	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12501	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12502	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12503	if (fIOWrite)
12504	{
12505	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12507	}
12508	else
12509	{
12510	uint32_t u32Result = 0;
12511	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12512	if (IOM_SUCCESS(rcStrict))
12513	{
12514	/* Save result of I/O IN instr. in AL/AX/EAX. */
12515	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12516	}
12517	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12518	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12519	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12520	}
12521	}
12522
12523	if (IOM_SUCCESS(rcStrict))
12524	{
12525	if (!fUpdateRipAlready)
12526	{
12527	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12528	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12529	}
12530
12531	/*
12532	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12533	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12534	*/
12535	if (fIOString)
12536	{
12537	/** @todo Single-step for INS/OUTS with REP prefix? */
12538	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12539	}
12540	else if ( !fDbgStepping
12541	&& fGstStepping)
12542	{
12543	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12544	}
12545
12546	/*
12547	* If any I/O breakpoints are armed, we need to check if one triggered
12548	* and take appropriate action.
12549	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12550	*/
12551	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12552	AssertRCReturn(rc2, rc2);
12553
12554	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12555	* execution engines about whether hyper BPs and such are pending. */
12556	uint32_t const uDr7 = pMixedCtx->dr[7];
12557	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12558	&& X86_DR7_ANY_RW_IO(uDr7)
12559	&& (pMixedCtx->cr4 & X86_CR4_DE))
12560	\|\| DBGFBpIsHwIoArmed(pVM)))
12561	{
12562	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12563
12564	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12565	VMMRZCallRing3Disable(pVCpu);
12566	HM_DISABLE_PREEMPT();
12567
12568	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12569
12570	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12571	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12572	{
12573	/* Raise #DB. */
12574	if (fIsGuestDbgActive)
12575	ASMSetDR6(pMixedCtx->dr[6]);
12576	if (pMixedCtx->dr[7] != uDr7)
12577	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12578
12579	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12580	}
12581	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12582	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12583	else if ( rcStrict2 != VINF_SUCCESS
12584	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12585	rcStrict = rcStrict2;
12586	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12587
12588	HM_RESTORE_PREEMPT();
12589	VMMRZCallRing3Enable(pVCpu);
12590	}
12591	}
12592
12593	#ifdef VBOX_STRICT
12594	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12595	Assert(!fIOWrite);
12596	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12597	Assert(fIOWrite);
12598	else
12599	{
12600	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12601	* statuses, that the VMM device and some others may return. See
12602	* IOM_SUCCESS() for guidance. */
12603	AssertMsg( RT_FAILURE(rcStrict)
12604	\|\| rcStrict == VINF_SUCCESS
12605	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12606	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12607	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12608	\|\| rcStrict == VINF_EM_RAW_TO_R3
12609	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12610	#endif
12611	}
12612	#endif
12613
12614	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12615	return rcStrict;
12616	}
12617
12618
12619	/**
12620	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12621	* VM-exit.
12622	*/
12623	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12624	{
12625	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12626
12627	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12628	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12629	AssertRCReturn(rc, rc);
12630	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12631	{
12632	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12633	AssertRCReturn(rc, rc);
12634	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12635	{
12636	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12637
12638	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12639	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12640
12641	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12642	Assert(!pVCpu->hm.s.Event.fPending);
12643	pVCpu->hm.s.Event.fPending = true;
12644	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12645	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12646	AssertRCReturn(rc, rc);
12647	if (fErrorCodeValid)
12648	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12649	else
12650	pVCpu->hm.s.Event.u32ErrCode = 0;
12651	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12652	&& uVector == X86_XCPT_PF)
12653	{
12654	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12655	}
12656
12657	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12659	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12660	}
12661	}
12662
12663	/* Fall back to the interpreter to emulate the task-switch. */
12664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12665	return VERR_EM_INTERPRETER;
12666	}
12667
12668
12669	/**
12670	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12671	*/
12672	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12673	{
12674	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12675	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12676	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12677	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12678	AssertRCReturn(rc, rc);
12679	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12680	return VINF_EM_DBG_STEPPED;
12681	}
12682
12683
12684	/**
12685	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12686	*/
12687	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12688	{
12689	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12690
12691	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12692
12693	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12694	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12695	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12696	{
12697	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12698	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12699	{
12700	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12701	return VERR_EM_INTERPRETER;
12702	}
12703	}
12704	else
12705	{
12706	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12707	rcStrict1 = VINF_SUCCESS;
12708	return rcStrict1;
12709	}
12710
12711	#if 0
12712	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12713	* just sync the whole thing. */
12714	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12715	#else
12716	/* Aggressive state sync. for now. */
12717	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12718	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12719	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12720	#endif
12721	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12722	AssertRCReturn(rc, rc);
12723
12724	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12725	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12726	VBOXSTRICTRC rcStrict2;
12727	switch (uAccessType)
12728	{
12729	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12730	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12731	{
12732	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12733	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12734	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12735
12736	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12737	GCPhys &= PAGE_BASE_GC_MASK;
12738	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12739	PVM pVM = pVCpu->CTX_SUFF(pVM);
12740	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12741	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12742
12743	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12744	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12745	CPUMCTX2CORE(pMixedCtx), GCPhys);
12746	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12747	if ( rcStrict2 == VINF_SUCCESS
12748	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12749	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12750	{
12751	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12752	\| HM_CHANGED_GUEST_RSP
12753	\| HM_CHANGED_GUEST_RFLAGS
12754	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
12755	rcStrict2 = VINF_SUCCESS;
12756	}
12757	break;
12758	}
12759
12760	default:
12761	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12762	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12763	break;
12764	}
12765
12766	if (rcStrict2 != VINF_SUCCESS)
12767	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12768	return rcStrict2;
12769	}
12770
12771
12772	/**
12773	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12774	* VM-exit.
12775	*/
12776	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12777	{
12778	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12779
12780	/* We should -not- get this VM-exit if the guest's debug registers were active. */
12781	if (pVmxTransient->fWasGuestDebugStateActive)
12782	{
12783	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12784	HMVMX_RETURN_UNEXPECTED_EXIT();
12785	}
12786
12787	if ( !pVCpu->hm.s.fSingleInstruction
12788	&& !pVmxTransient->fWasHyperDebugStateActive)
12789	{
12790	Assert(!DBGFIsStepping(pVCpu));
12791	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12792
12793	/* Don't intercept MOV DRx any more. */
12794	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12795	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12796	AssertRCReturn(rc, rc);
12797
12798	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12799	VMMRZCallRing3Disable(pVCpu);
12800	HM_DISABLE_PREEMPT();
12801
12802	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12803	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12804	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
12805
12806	HM_RESTORE_PREEMPT();
12807	VMMRZCallRing3Enable(pVCpu);
12808
12809	#ifdef VBOX_WITH_STATISTICS
12810	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12811	AssertRCReturn(rc, rc);
12812	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12813	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12814	else
12815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12816	#endif
12817	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12818	return VINF_SUCCESS;
12819	}
12820
12821	/*
12822	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12823	* Update the segment registers and DR7 from the CPU.
12824	*/
12825	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12826	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12827	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12828	AssertRCReturn(rc, rc);
12829	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12830
12831	PVM pVM = pVCpu->CTX_SUFF(pVM);
12832	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12833	{
12834	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12835	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12836	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12837	if (RT_SUCCESS(rc))
12838	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12840	}
12841	else
12842	{
12843	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12844	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12845	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12847	}
12848
12849	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
12850	if (RT_SUCCESS(rc))
12851	{
12852	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12853	AssertRCReturn(rc2, rc2);
12854	return VINF_SUCCESS;
12855	}
12856	return rc;
12857	}
12858
12859
12860	/**
12861	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12862	* Conditional VM-exit.
12863	*/
12864	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12865	{
12866	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12867	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12868
12869	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12870	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12871	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12872	{
12873	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12874	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12875	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12876	{
12877	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12878	return VERR_EM_INTERPRETER;
12879	}
12880	}
12881	else
12882	{
12883	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12884	rcStrict1 = VINF_SUCCESS;
12885	return rcStrict1;
12886	}
12887
12888	RTGCPHYS GCPhys = 0;
12889	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12890
12891	#if 0
12892	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12893	#else
12894	/* Aggressive state sync. for now. */
12895	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12896	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12897	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12898	#endif
12899	AssertRCReturn(rc, rc);
12900
12901	/*
12902	* If we succeed, resume guest execution.
12903	* If we fail in interpreting the instruction because we couldn't get the guest physical address
12904	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12905	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12906	* weird case. See @bugref{6043}.
12907	*/
12908	PVM pVM = pVCpu->CTX_SUFF(pVM);
12909	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12910	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12911	if ( rcStrict2 == VINF_SUCCESS
12912	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12913	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12914	{
12915	/* Successfully handled MMIO operation. */
12916	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12917	\| HM_CHANGED_GUEST_RSP
12918	\| HM_CHANGED_GUEST_RFLAGS
12919	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
12920	return VINF_SUCCESS;
12921	}
12922	return rcStrict2;
12923	}
12924
12925
12926	/**
12927	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12928	* VM-exit.
12929	*/
12930	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12931	{
12932	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12933	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12934
12935	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12936	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12937	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12938	{
12939	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12940	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12941	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12942	}
12943	else
12944	{
12945	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12946	rcStrict1 = VINF_SUCCESS;
12947	return rcStrict1;
12948	}
12949
12950	RTGCPHYS GCPhys = 0;
12951	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12952	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12953	#if 0
12954	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12955	#else
12956	/* Aggressive state sync. for now. */
12957	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12958	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12959	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12960	#endif
12961	AssertRCReturn(rc, rc);
12962
12963	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12964	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12965
12966	RTGCUINT uErrorCode = 0;
12967	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12968	uErrorCode \|= X86_TRAP_PF_ID;
12969	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12970	uErrorCode \|= X86_TRAP_PF_RW;
12971	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12972	uErrorCode \|= X86_TRAP_PF_P;
12973
12974	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12975
12976	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12977	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12978
12979	/* Handle the pagefault trap for the nested shadow table. */
12980	PVM pVM = pVCpu->CTX_SUFF(pVM);
12981	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12982	TRPMResetTrap(pVCpu);
12983
12984	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12985	if ( rcStrict2 == VINF_SUCCESS
12986	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12987	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
12988	{
12989	/* Successfully synced our nested page tables. */
12990	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12991	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12992	\| HM_CHANGED_GUEST_RSP
12993	\| HM_CHANGED_GUEST_RFLAGS);
12994	return VINF_SUCCESS;
12995	}
12996
12997	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12998	return rcStrict2;
12999	}
13000
13001	/** @} */
13002
13003	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13004	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13005	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13006
13007	/** @name VM-exit exception handlers.
13008	* @{
13009	*/
13010
13011	/**
13012	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13013	*/
13014	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13015	{
13016	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13018
13019	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13020	AssertRCReturn(rc, rc);
13021
13022	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13023	{
13024	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13025	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13026
13027	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13028	* provides VM-exit instruction length. If this causes problem later,
13029	* disassemble the instruction like it's done on AMD-V. */
13030	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13031	AssertRCReturn(rc2, rc2);
13032	return rc;
13033	}
13034
13035	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13036	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13037	return rc;
13038	}
13039
13040
13041	/**
13042	* VM-exit exception handler for \#BP (Breakpoint exception).
13043	*/
13044	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13045	{
13046	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13047	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13048
13049	/** @todo Try optimize this by not saving the entire guest state unless
13050	* really needed. */
13051	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13052	AssertRCReturn(rc, rc);
13053
13054	PVM pVM = pVCpu->CTX_SUFF(pVM);
13055	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13056	if (rc == VINF_EM_RAW_GUEST_TRAP)
13057	{
13058	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13059	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13060	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13061	AssertRCReturn(rc, rc);
13062
13063	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13064	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13065	}
13066
13067	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13068	return rc;
13069	}
13070
13071
13072	/**
13073	* VM-exit exception handler for \#AC (alignment check exception).
13074	*/
13075	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13076	{
13077	RT_NOREF_PV(pMixedCtx);
13078	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13079
13080	/*
13081	* Re-inject it. We'll detect any nesting before getting here.
13082	*/
13083	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13084	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13085	AssertRCReturn(rc, rc);
13086	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13087
13088	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13089	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13090	return VINF_SUCCESS;
13091	}
13092
13093
13094	/**
13095	* VM-exit exception handler for \#DB (Debug exception).
13096	*/
13097	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13098	{
13099	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13101	Log6(("XcptDB\n"));
13102
13103	/*
13104	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13105	* for processing.
13106	*/
13107	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13108	AssertRCReturn(rc, rc);
13109
13110	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13111	uint64_t uDR6 = X86_DR6_INIT_VAL;
13112	uDR6 \|= ( pVmxTransient->uExitQualification
13113	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13114
13115	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13116	if (rc == VINF_EM_RAW_GUEST_TRAP)
13117	{
13118	/*
13119	* The exception was for the guest. Update DR6, DR7.GD and
13120	* IA32_DEBUGCTL.LBR before forwarding it.
13121	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13122	*/
13123	VMMRZCallRing3Disable(pVCpu);
13124	HM_DISABLE_PREEMPT();
13125
13126	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13127	pMixedCtx->dr[6] \|= uDR6;
13128	if (CPUMIsGuestDebugStateActive(pVCpu))
13129	ASMSetDR6(pMixedCtx->dr[6]);
13130
13131	HM_RESTORE_PREEMPT();
13132	VMMRZCallRing3Enable(pVCpu);
13133
13134	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13135	AssertRCReturn(rc, rc);
13136
13137	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13138	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13139
13140	/* Paranoia. */
13141	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13142	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13143
13144	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13145	AssertRCReturn(rc, rc);
13146
13147	/*
13148	* Raise #DB in the guest.
13149	*
13150	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13151	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13152	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13153	*
13154	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13155	*/
13156	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13157	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13158	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13159	AssertRCReturn(rc, rc);
13160	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13161	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13162	return VINF_SUCCESS;
13163	}
13164
13165	/*
13166	* Not a guest trap, must be a hypervisor related debug event then.
13167	* Update DR6 in case someone is interested in it.
13168	*/
13169	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13170	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13171	CPUMSetHyperDR6(pVCpu, uDR6);
13172
13173	return rc;
13174	}
13175
13176
13177	/**
13178	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13179	* point exception).
13180	*/
13181	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13182	{
13183	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13184
13185	/* We require CR0 and EFER. EFER is always up-to-date. */
13186	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13187	AssertRCReturn(rc, rc);
13188
13189	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13190	VMMRZCallRing3Disable(pVCpu);
13191	HM_DISABLE_PREEMPT();
13192
13193	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13194	if (pVmxTransient->fWasGuestFPUStateActive)
13195	{
13196	rc = VINF_EM_RAW_GUEST_TRAP;
13197	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13198	}
13199	else
13200	{
13201	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13202	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13203	#endif
13204	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13205	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13206	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13207	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13208	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13209	}
13210
13211	HM_RESTORE_PREEMPT();
13212	VMMRZCallRing3Enable(pVCpu);
13213
13214	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13215	{
13216	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13217	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13218	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13219	pVCpu->hm.s.fPreloadGuestFpu = true;
13220	}
13221	else
13222	{
13223	/* Forward #NM to the guest. */
13224	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13225	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13226	AssertRCReturn(rc, rc);
13227	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13228	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13229	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13230	}
13231
13232	return VINF_SUCCESS;
13233	}
13234
13235
13236	/**
13237	* VM-exit exception handler for \#GP (General-protection exception).
13238	*
13239	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13240	*/
13241	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13242	{
13243	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13244	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13245
13246	int rc;
13247	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13248	{ /* likely */ }
13249	else
13250	{
13251	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13252	Assert(pVCpu->hm.s.fUsingDebugLoop);
13253	#endif
13254	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13255	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13256	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13257	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13258	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13259	AssertRCReturn(rc, rc);
13260	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13261	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13262	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13263	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13264	return rc;
13265	}
13266
13267	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13268	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13269
13270	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13271	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13272	AssertRCReturn(rc, rc);
13273
13274	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13275	uint32_t cbOp = 0;
13276	PVM pVM = pVCpu->CTX_SUFF(pVM);
13277	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13278	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13279	if (RT_SUCCESS(rc))
13280	{
13281	rc = VINF_SUCCESS;
13282	Assert(cbOp == pDis->cbInstr);
13283	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13284	switch (pDis->pCurInstr->uOpcode)
13285	{
13286	case OP_CLI:
13287	{
13288	pMixedCtx->eflags.Bits.u1IF = 0;
13289	pMixedCtx->eflags.Bits.u1RF = 0;
13290	pMixedCtx->rip += pDis->cbInstr;
13291	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13292	if ( !fDbgStepping
13293	&& pMixedCtx->eflags.Bits.u1TF)
13294	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13296	break;
13297	}
13298
13299	case OP_STI:
13300	{
13301	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13302	pMixedCtx->eflags.Bits.u1IF = 1;
13303	pMixedCtx->eflags.Bits.u1RF = 0;
13304	pMixedCtx->rip += pDis->cbInstr;
13305	if (!fOldIF)
13306	{
13307	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13308	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13309	}
13310	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13311	if ( !fDbgStepping
13312	&& pMixedCtx->eflags.Bits.u1TF)
13313	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13315	break;
13316	}
13317
13318	case OP_HLT:
13319	{
13320	rc = VINF_EM_HALT;
13321	pMixedCtx->rip += pDis->cbInstr;
13322	pMixedCtx->eflags.Bits.u1RF = 0;
13323	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13325	break;
13326	}
13327
13328	case OP_POPF:
13329	{
13330	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13331	uint32_t cbParm;
13332	uint32_t uMask;
13333	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13334	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13335	{
13336	cbParm = 4;
13337	uMask = 0xffffffff;
13338	}
13339	else
13340	{
13341	cbParm = 2;
13342	uMask = 0xffff;
13343	}
13344
13345	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13346	RTGCPTR GCPtrStack = 0;
13347	X86EFLAGS Eflags;
13348	Eflags.u32 = 0;
13349	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13350	&GCPtrStack);
13351	if (RT_SUCCESS(rc))
13352	{
13353	Assert(sizeof(Eflags.u32) >= cbParm);
13354	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13355	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13356	}
13357	if (RT_FAILURE(rc))
13358	{
13359	rc = VERR_EM_INTERPRETER;
13360	break;
13361	}
13362	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13363	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13364	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13365	pMixedCtx->esp += cbParm;
13366	pMixedCtx->esp &= uMask;
13367	pMixedCtx->rip += pDis->cbInstr;
13368	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13369	\| HM_CHANGED_GUEST_RSP
13370	\| HM_CHANGED_GUEST_RFLAGS);
13371	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13372	POPF restores EFLAGS.TF. */
13373	if ( !fDbgStepping
13374	&& fGstStepping)
13375	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13377	break;
13378	}
13379
13380	case OP_PUSHF:
13381	{
13382	uint32_t cbParm;
13383	uint32_t uMask;
13384	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13385	{
13386	cbParm = 4;
13387	uMask = 0xffffffff;
13388	}
13389	else
13390	{
13391	cbParm = 2;
13392	uMask = 0xffff;
13393	}
13394
13395	/* Get the stack pointer & push the contents of eflags onto the stack. */
13396	RTGCPTR GCPtrStack = 0;
13397	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13398	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13399	if (RT_FAILURE(rc))
13400	{
13401	rc = VERR_EM_INTERPRETER;
13402	break;
13403	}
13404	X86EFLAGS Eflags = pMixedCtx->eflags;
13405	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13406	Eflags.Bits.u1RF = 0;
13407	Eflags.Bits.u1VM = 0;
13408
13409	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13410	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13411	{
13412	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13413	rc = VERR_EM_INTERPRETER;
13414	break;
13415	}
13416	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13417	pMixedCtx->esp -= cbParm;
13418	pMixedCtx->esp &= uMask;
13419	pMixedCtx->rip += pDis->cbInstr;
13420	pMixedCtx->eflags.Bits.u1RF = 0;
13421	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13422	\| HM_CHANGED_GUEST_RSP
13423	\| HM_CHANGED_GUEST_RFLAGS);
13424	if ( !fDbgStepping
13425	&& pMixedCtx->eflags.Bits.u1TF)
13426	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13427	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13428	break;
13429	}
13430
13431	case OP_IRET:
13432	{
13433	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13434	* instruction reference. */
13435	RTGCPTR GCPtrStack = 0;
13436	uint32_t uMask = 0xffff;
13437	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13438	uint16_t aIretFrame[3];
13439	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13440	{
13441	rc = VERR_EM_INTERPRETER;
13442	break;
13443	}
13444	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13445	&GCPtrStack);
13446	if (RT_SUCCESS(rc))
13447	{
13448	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13449	PGMACCESSORIGIN_HM));
13450	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13451	}
13452	if (RT_FAILURE(rc))
13453	{
13454	rc = VERR_EM_INTERPRETER;
13455	break;
13456	}
13457	pMixedCtx->eip = 0;
13458	pMixedCtx->ip = aIretFrame[0];
13459	pMixedCtx->cs.Sel = aIretFrame[1];
13460	pMixedCtx->cs.ValidSel = aIretFrame[1];
13461	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13462	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13463	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13464	pMixedCtx->sp += sizeof(aIretFrame);
13465	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13466	\| HM_CHANGED_GUEST_SEGMENT_REGS
13467	\| HM_CHANGED_GUEST_RSP
13468	\| HM_CHANGED_GUEST_RFLAGS);
13469	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13470	if ( !fDbgStepping
13471	&& fGstStepping)
13472	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13473	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13474	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13475	break;
13476	}
13477
13478	case OP_INT:
13479	{
13480	uint16_t uVector = pDis->Param1.uValue & 0xff;
13481	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13482	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13484	break;
13485	}
13486
13487	case OP_INTO:
13488	{
13489	if (pMixedCtx->eflags.Bits.u1OF)
13490	{
13491	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13492	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13494	}
13495	else
13496	{
13497	pMixedCtx->eflags.Bits.u1RF = 0;
13498	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13499	}
13500	break;
13501	}
13502
13503	default:
13504	{
13505	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13506	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13507	EMCODETYPE_SUPERVISOR);
13508	rc = VBOXSTRICTRC_VAL(rc2);
13509	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13510	/** @todo We have to set pending-debug exceptions here when the guest is
13511	* single-stepping depending on the instruction that was interpreted. */
13512	Log4(("#GP rc=%Rrc\n", rc));
13513	break;
13514	}
13515	}
13516	}
13517	else
13518	rc = VERR_EM_INTERPRETER;
13519
13520	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13521	("#GP Unexpected rc=%Rrc\n", rc));
13522	return rc;
13523	}
13524
13525
13526	/**
13527	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13528	* the exception reported in the VMX transient structure back into the VM.
13529	*
13530	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13531	* up-to-date.
13532	*/
13533	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13534	{
13535	RT_NOREF_PV(pMixedCtx);
13536	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13537	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13538	Assert(pVCpu->hm.s.fUsingDebugLoop);
13539	#endif
13540
13541	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13542	hmR0VmxCheckExitDueToEventDelivery(). */
13543	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13544	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13545	AssertRCReturn(rc, rc);
13546	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13547
13548	#ifdef DEBUG_ramshankar
13549	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13550	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13551	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13552	#endif
13553
13554	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13555	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13556	return VINF_SUCCESS;
13557	}
13558
13559
13560	/**
13561	* VM-exit exception handler for \#PF (Page-fault exception).
13562	*/
13563	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13564	{
13565	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13566	PVM pVM = pVCpu->CTX_SUFF(pVM);
13567	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13568	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13569	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13570	AssertRCReturn(rc, rc);
13571
13572	if (!pVM->hm.s.fNestedPaging)
13573	{ /* likely */ }
13574	else
13575	{
13576	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13577	Assert(pVCpu->hm.s.fUsingDebugLoop);
13578	#endif
13579	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13580	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13581	{
13582	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13583	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13584	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13585	}
13586	else
13587	{
13588	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13589	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13590	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13591	}
13592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13593	return rc;
13594	}
13595
13596	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13597	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13598	if (pVmxTransient->fVectoringPF)
13599	{
13600	Assert(pVCpu->hm.s.Event.fPending);
13601	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13602	}
13603
13604	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13605	AssertRCReturn(rc, rc);
13606
13607	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13608	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13609
13610	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13611	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13612	(RTGCPTR)pVmxTransient->uExitQualification);
13613
13614	Log4(("#PF: rc=%Rrc\n", rc));
13615	if (rc == VINF_SUCCESS)
13616	{
13617	#if 0
13618	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13619	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13620	* memory? We don't update the whole state here... */
13621	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13622	\| HM_CHANGED_GUEST_RSP
13623	\| HM_CHANGED_GUEST_RFLAGS
13624	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13625	#else
13626	/*
13627	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13628	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13629	*/
13630	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13631	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13632	#endif
13633	TRPMResetTrap(pVCpu);
13634	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13635	return rc;
13636	}
13637
13638	if (rc == VINF_EM_RAW_GUEST_TRAP)
13639	{
13640	if (!pVmxTransient->fVectoringDoublePF)
13641	{
13642	/* It's a guest page fault and needs to be reflected to the guest. */
13643	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13644	TRPMResetTrap(pVCpu);
13645	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13646	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13647	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13648	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13649	}
13650	else
13651	{
13652	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13653	TRPMResetTrap(pVCpu);
13654	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13655	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13656	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13657	}
13658
13659	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13660	return VINF_SUCCESS;
13661	}
13662
13663	TRPMResetTrap(pVCpu);
13664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13665	return rc;
13666	}
13667
13668	/** @} */
13669

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 64856

Download in other formats: