HMVMXR0.cpp@ 66706

Last change on this file since 66706 was 66706, checked in by vboxsync, 8 years ago
VMM/HMVMXR0: Disabled using IEM event reflection from r115095. Needs more debugging.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 596.8 KB

Line
1	/* $Id: HMVMXR0.cpp 66706 2017-04-27 17:56:12Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2016 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#ifdef DEBUG_ramshankar
44	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49	# define HMVMX_ALWAYS_TRAP_PF
50	# define HMVMX_ALWAYS_SWAP_FPU_STATE
51	# define HMVMX_ALWAYS_FLUSH_TLB
52	# define HMVMX_ALWAYS_SWAP_EFER
53	# define HMVMX_USE_IEM_EVENT_REFLECTION
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually except:
142	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144	* due to bugs in Intel CPUs.
145	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
146	* support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*********************************************************************************************************************************
197	* Structures and Typedefs *
198	*********************************************************************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG fEFlags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit code qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u7Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringDoublePF;
278	/** Whether the VM-exit was caused by a page-fault during delivery of an
279	* external interrupt or NMI. */
280	bool fVectoringPF;
281	} VMXTRANSIENT;
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
285	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
286	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
287	/** Pointer to VMX transient state. */
288	typedef VMXTRANSIENT *PVMXTRANSIENT;
289
290
291	/**
292	* MSR-bitmap read permissions.
293	*/
294	typedef enum VMXMSREXITREAD
295	{
296	/** Reading this MSR causes a VM-exit. */
297	VMXMSREXIT_INTERCEPT_READ = 0xb,
298	/** Reading this MSR does not cause a VM-exit. */
299	VMXMSREXIT_PASSTHRU_READ
300	} VMXMSREXITREAD;
301	/** Pointer to MSR-bitmap read permissions. */
302	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
303
304	/**
305	* MSR-bitmap write permissions.
306	*/
307	typedef enum VMXMSREXITWRITE
308	{
309	/** Writing to this MSR causes a VM-exit. */
310	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
311	/** Writing to this MSR does not cause a VM-exit. */
312	VMXMSREXIT_PASSTHRU_WRITE
313	} VMXMSREXITWRITE;
314	/** Pointer to MSR-bitmap write permissions. */
315	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
316
317
318	/**
319	* VMX VM-exit handler.
320	*
321	* @returns Strict VBox status code (i.e. informational status codes too).
322	* @param pVCpu The cross context virtual CPU structure.
323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
324	* out-of-sync. Make sure to update the required
325	* fields before using them.
326	* @param pVmxTransient Pointer to the VMX-transient structure.
327	*/
328	#ifndef HMVMX_USE_FUNCTION_TABLE
329	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330	#else
331	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332	/** Pointer to VM-exit handler. */
333	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
334	#endif
335
336	/**
337	* VMX VM-exit handler, non-strict status code.
338	*
339	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
340	*
341	* @returns VBox status code, no informational status code returned.
342	* @param pVCpu The cross context virtual CPU structure.
343	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
344	* out-of-sync. Make sure to update the required
345	* fields before using them.
346	* @param pVmxTransient Pointer to the VMX-transient structure.
347	*
348	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
349	* use of that status code will be replaced with VINF_EM_SOMETHING
350	* later when switching over to IEM.
351	*/
352	#ifndef HMVMX_USE_FUNCTION_TABLE
353	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
354	#else
355	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
356	#endif
357
358
359	/*********************************************************************************************************************************
360	* Internal Functions *
361	*********************************************************************************************************************************/
362	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
363	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
364	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
365	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
366	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
367	bool fStepping, uint32_t *puIntState);
368	#if HC_ARCH_BITS == 32
369	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
370	#endif
371	#ifndef HMVMX_USE_FUNCTION_TABLE
372	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
373	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
374	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
375	#else
376	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
377	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
378	#endif
379
380
381	/** @name VM-exit handlers.
382	* @{
383	*/
384	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396	static FNVMXEXITHANDLER hmR0VmxExitHlt;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
403	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
404	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
405	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
406	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
407	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
408	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
412	static FNVMXEXITHANDLER hmR0VmxExitMwait;
413	static FNVMXEXITHANDLER hmR0VmxExitMtf;
414	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
415	static FNVMXEXITHANDLER hmR0VmxExitPause;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
417	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
418	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
419	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
422	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
423	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
424	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
425	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
426	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
427	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
428	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
429	/** @} */
430
431	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
440
441
442	/*********************************************************************************************************************************
443	* Global Variables *
444	*********************************************************************************************************************************/
445	#ifdef HMVMX_USE_FUNCTION_TABLE
446
447	/**
448	* VMX_EXIT dispatch table.
449	*/
450	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
451	{
452	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
453	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
454	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
455	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
456	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
457	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
458	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
459	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
460	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
461	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
462	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
463	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
464	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
465	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
466	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
467	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
468	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
469	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
470	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
471	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
472	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
473	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
474	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
475	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
476	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
477	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
478	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
479	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
480	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
481	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
482	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
483	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
484	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
485	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
486	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
487	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
488	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
489	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
490	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
491	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
492	/* 40 UNDEFINED */ hmR0VmxExitPause,
493	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
494	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
495	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
496	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
497	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
498	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
499	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
500	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
501	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
502	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
503	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
504	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
505	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
506	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
507	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
508	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
509	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
510	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
511	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
512	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
513	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
514	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
515	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
516	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
517	};
518	#endif /* HMVMX_USE_FUNCTION_TABLE */
519
520	#ifdef VBOX_STRICT
521	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
522	{
523	/* 0 */ "(Not Used)",
524	/* 1 */ "VMCALL executed in VMX root operation.",
525	/* 2 */ "VMCLEAR with invalid physical address.",
526	/* 3 */ "VMCLEAR with VMXON pointer.",
527	/* 4 */ "VMLAUNCH with non-clear VMCS.",
528	/* 5 */ "VMRESUME with non-launched VMCS.",
529	/* 6 */ "VMRESUME after VMXOFF",
530	/* 7 */ "VM-entry with invalid control fields.",
531	/* 8 */ "VM-entry with invalid host state fields.",
532	/* 9 */ "VMPTRLD with invalid physical address.",
533	/* 10 */ "VMPTRLD with VMXON pointer.",
534	/* 11 */ "VMPTRLD with incorrect revision identifier.",
535	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
536	/* 13 */ "VMWRITE to read-only VMCS component.",
537	/* 14 */ "(Not Used)",
538	/* 15 */ "VMXON executed in VMX root operation.",
539	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
540	/* 17 */ "VM-entry with non-launched executing VMCS.",
541	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
542	/* 19 */ "VMCALL with non-clear VMCS.",
543	/* 20 */ "VMCALL with invalid VM-exit control fields.",
544	/* 21 */ "(Not Used)",
545	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
546	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
547	/* 24 */ "VMCALL with invalid SMM-monitor features.",
548	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
549	/* 26 */ "VM-entry with events blocked by MOV SS.",
550	/* 27 */ "(Not Used)",
551	/* 28 */ "Invalid operand to INVEPT/INVVPID."
552	};
553	#endif /* VBOX_STRICT */
554
555
556
557	/**
558	* Updates the VM's last error record.
559	*
560	* If there was a VMX instruction error, reads the error data from the VMCS and
561	* updates VCPU's last error record as well.
562	*
563	* @param pVM The cross context VM structure.
564	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
565	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
566	* VERR_VMX_INVALID_VMCS_FIELD.
567	* @param rc The error code.
568	*/
569	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
570	{
571	AssertPtr(pVM);
572	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
573	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
574	{
575	AssertPtrReturnVoid(pVCpu);
576	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
577	}
578	pVM->hm.s.lLastError = rc;
579	}
580
581
582	/**
583	* Reads the VM-entry interruption-information field from the VMCS into the VMX
584	* transient structure.
585	*
586	* @returns VBox status code.
587	* @param pVmxTransient Pointer to the VMX transient structure.
588	*
589	* @remarks No-long-jump zone!!!
590	*/
591	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
592	{
593	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
594	AssertRCReturn(rc, rc);
595	return VINF_SUCCESS;
596	}
597
598
599	#ifdef VBOX_STRICT
600	/**
601	* Reads the VM-entry exception error code field from the VMCS into
602	* the VMX transient structure.
603	*
604	* @returns VBox status code.
605	* @param pVmxTransient Pointer to the VMX transient structure.
606	*
607	* @remarks No-long-jump zone!!!
608	*/
609	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612	AssertRCReturn(rc, rc);
613	return VINF_SUCCESS;
614	}
615	#endif /* VBOX_STRICT */
616
617
618	#ifdef VBOX_STRICT
619	/**
620	* Reads the VM-entry exception error code field from the VMCS into
621	* the VMX transient structure.
622	*
623	* @returns VBox status code.
624	* @param pVmxTransient Pointer to the VMX transient structure.
625	*
626	* @remarks No-long-jump zone!!!
627	*/
628	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
629	{
630	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
631	AssertRCReturn(rc, rc);
632	return VINF_SUCCESS;
633	}
634	#endif /* VBOX_STRICT */
635
636
637	/**
638	* Reads the VM-exit interruption-information field from the VMCS into the VMX
639	* transient structure.
640	*
641	* @returns VBox status code.
642	* @param pVmxTransient Pointer to the VMX transient structure.
643	*/
644	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
645	{
646	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
647	{
648	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
649	AssertRCReturn(rc, rc);
650	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
651	}
652	return VINF_SUCCESS;
653	}
654
655
656	/**
657	* Reads the VM-exit interruption error code from the VMCS into the VMX
658	* transient structure.
659	*
660	* @returns VBox status code.
661	* @param pVmxTransient Pointer to the VMX transient structure.
662	*/
663	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
664	{
665	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
666	{
667	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
668	AssertRCReturn(rc, rc);
669	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
670	}
671	return VINF_SUCCESS;
672	}
673
674
675	/**
676	* Reads the VM-exit instruction length field from the VMCS into the VMX
677	* transient structure.
678	*
679	* @returns VBox status code.
680	* @param pVmxTransient Pointer to the VMX transient structure.
681	*/
682	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
683	{
684	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
685	{
686	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
687	AssertRCReturn(rc, rc);
688	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
689	}
690	return VINF_SUCCESS;
691	}
692
693
694	/**
695	* Reads the VM-exit instruction-information field from the VMCS into
696	* the VMX transient structure.
697	*
698	* @returns VBox status code.
699	* @param pVmxTransient Pointer to the VMX transient structure.
700	*/
701	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
702	{
703	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
704	{
705	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
706	AssertRCReturn(rc, rc);
707	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
708	}
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Reads the exit code qualification from the VMCS into the VMX transient
715	* structure.
716	*
717	* @returns VBox status code.
718	* @param pVCpu The cross context virtual CPU structure of the
719	* calling EMT. (Required for the VMCS cache case.)
720	* @param pVmxTransient Pointer to the VMX transient structure.
721	*/
722	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
723	{
724	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
725	{
726	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
727	AssertRCReturn(rc, rc);
728	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
729	}
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Reads the IDT-vectoring information field from the VMCS into the VMX
736	* transient structure.
737	*
738	* @returns VBox status code.
739	* @param pVmxTransient Pointer to the VMX transient structure.
740	*
741	* @remarks No-long-jump zone!!!
742	*/
743	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
744	{
745	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
746	{
747	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
748	AssertRCReturn(rc, rc);
749	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
750	}
751	return VINF_SUCCESS;
752	}
753
754
755	/**
756	* Reads the IDT-vectoring error code from the VMCS into the VMX
757	* transient structure.
758	*
759	* @returns VBox status code.
760	* @param pVmxTransient Pointer to the VMX transient structure.
761	*/
762	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
763	{
764	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
765	{
766	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
767	AssertRCReturn(rc, rc);
768	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
769	}
770	return VINF_SUCCESS;
771	}
772
773
774	/**
775	* Enters VMX root mode operation on the current CPU.
776	*
777	* @returns VBox status code.
778	* @param pVM The cross context VM structure. Can be
779	* NULL, after a resume.
780	* @param HCPhysCpuPage Physical address of the VMXON region.
781	* @param pvCpuPage Pointer to the VMXON region.
782	*/
783	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
784	{
785	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
786	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
787	Assert(pvCpuPage);
788	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
789
790	if (pVM)
791	{
792	/* Write the VMCS revision dword to the VMXON region. */
793	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
794	}
795
796	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
797	RTCCUINTREG fEFlags = ASMIntDisableFlags();
798
799	/* Enable the VMX bit in CR4 if necessary. */
800	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
801
802	/* Enter VMX root mode. */
803	int rc = VMXEnable(HCPhysCpuPage);
804	if (RT_FAILURE(rc))
805	{
806	if (!(uOldCr4 & X86_CR4_VMXE))
807	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
808
809	if (pVM)
810	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
811	}
812
813	/* Restore interrupts. */
814	ASMSetFlags(fEFlags);
815	return rc;
816	}
817
818
819	/**
820	* Exits VMX root mode operation on the current CPU.
821	*
822	* @returns VBox status code.
823	*/
824	static int hmR0VmxLeaveRootMode(void)
825	{
826	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
827
828	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
829	RTCCUINTREG fEFlags = ASMIntDisableFlags();
830
831	/* If we're for some reason not in VMX root mode, then don't leave it. */
832	RTCCUINTREG uHostCR4 = ASMGetCR4();
833
834	int rc;
835	if (uHostCR4 & X86_CR4_VMXE)
836	{
837	/* Exit VMX root mode and clear the VMX bit in CR4. */
838	VMXDisable();
839	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
840	rc = VINF_SUCCESS;
841	}
842	else
843	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
844
845	/* Restore interrupts. */
846	ASMSetFlags(fEFlags);
847	return rc;
848	}
849
850
851	/**
852	* Allocates and maps one physically contiguous page. The allocated page is
853	* zero'd out. (Used by various VT-x structures).
854	*
855	* @returns IPRT status code.
856	* @param pMemObj Pointer to the ring-0 memory object.
857	* @param ppVirt Where to store the virtual address of the
858	* allocation.
859	* @param pHCPhys Where to store the physical address of the
860	* allocation.
861	*/
862	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
863	{
864	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
865	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
866	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
867
868	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
869	if (RT_FAILURE(rc))
870	return rc;
871	ppVirt = RTR0MemObjAddress(pMemObj);
872	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
873	ASMMemZero32(*ppVirt, PAGE_SIZE);
874	return VINF_SUCCESS;
875	}
876
877
878	/**
879	* Frees and unmaps an allocated physical page.
880	*
881	* @param pMemObj Pointer to the ring-0 memory object.
882	* @param ppVirt Where to re-initialize the virtual address of
883	* allocation as 0.
884	* @param pHCPhys Where to re-initialize the physical address of the
885	* allocation as 0.
886	*/
887	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
888	{
889	AssertPtr(pMemObj);
890	AssertPtr(ppVirt);
891	AssertPtr(pHCPhys);
892	if (*pMemObj != NIL_RTR0MEMOBJ)
893	{
894	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
895	AssertRC(rc);
896	*pMemObj = NIL_RTR0MEMOBJ;
897	*ppVirt = 0;
898	*pHCPhys = 0;
899	}
900	}
901
902
903	/**
904	* Worker function to free VT-x related structures.
905	*
906	* @returns IPRT status code.
907	* @param pVM The cross context VM structure.
908	*/
909	static void hmR0VmxStructsFree(PVM pVM)
910	{
911	for (VMCPUID i = 0; i < pVM->cCpus; i++)
912	{
913	PVMCPU pVCpu = &pVM->aCpus[i];
914	AssertPtr(pVCpu);
915
916	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
917	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
918
919	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
921
922	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923	}
924
925	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928	#endif
929	}
930
931
932	/**
933	* Worker function to allocate VT-x related VM structures.
934	*
935	* @returns IPRT status code.
936	* @param pVM The cross context VM structure.
937	*/
938	static int hmR0VmxStructsAlloc(PVM pVM)
939	{
940	/*
941	* Initialize members up-front so we can cleanup properly on allocation failure.
942	*/
943	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946	pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955	#endif
956	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959	for (VMCPUID i = 0; i < pVM->cCpus; i++)
960	{
961	PVMCPU pVCpu = &pVM->aCpus[i];
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
965	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
966	}
967	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
968	#undef VMXLOCAL_INIT_VM_MEMOBJ
969
970	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
971	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
972	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
973	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
974
975	/*
976	* Allocate all the VT-x structures.
977	*/
978	int rc = VINF_SUCCESS;
979	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
980	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
981	if (RT_FAILURE(rc))
982	goto cleanup;
983	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
984	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
985	#endif
986
987	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
988	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
989	{
990	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
991	&pVM->hm.s.vmx.HCPhysApicAccess);
992	if (RT_FAILURE(rc))
993	goto cleanup;
994	}
995
996	/*
997	* Initialize per-VCPU VT-x structures.
998	*/
999	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1000	{
1001	PVMCPU pVCpu = &pVM->aCpus[i];
1002	AssertPtr(pVCpu);
1003
1004	/* Allocate the VM control structure (VMCS). */
1005	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1006	if (RT_FAILURE(rc))
1007	goto cleanup;
1008
1009	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1010	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1011	{
1012	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013	NULL /* pR3Ptr /, NULL / pRCPtr */);
1014	if (RT_FAILURE(rc))
1015	goto cleanup;
1016	}
1017
1018	/*
1019	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020	* transparent accesses of specific MSRs.
1021	*
1022	* If the condition for enabling MSR bitmaps changes here, don't forget to
1023	* update HMAreMsrBitmapsAvailable().
1024	*/
1025	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026	{
1027	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029	if (RT_FAILURE(rc))
1030	goto cleanup;
1031	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032	}
1033
1034	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036	if (RT_FAILURE(rc))
1037	goto cleanup;
1038
1039	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1040	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041	if (RT_FAILURE(rc))
1042	goto cleanup;
1043	}
1044
1045	return VINF_SUCCESS;
1046
1047	cleanup:
1048	hmR0VmxStructsFree(pVM);
1049	return rc;
1050	}
1051
1052
1053	/**
1054	* Does global VT-x initialization (called during module initialization).
1055	*
1056	* @returns VBox status code.
1057	*/
1058	VMMR0DECL(int) VMXR0GlobalInit(void)
1059	{
1060	#ifdef HMVMX_USE_FUNCTION_TABLE
1061	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062	# ifdef VBOX_STRICT
1063	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064	Assert(g_apfnVMExitHandlers[i]);
1065	# endif
1066	#endif
1067	return VINF_SUCCESS;
1068	}
1069
1070
1071	/**
1072	* Does global VT-x termination (called during module termination).
1073	*/
1074	VMMR0DECL(void) VMXR0GlobalTerm()
1075	{
1076	/* Nothing to do currently. */
1077	}
1078
1079
1080	/**
1081	* Sets up and activates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pVM The cross context VM structure. Can be
1086	* NULL after a host resume operation.
1087	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088	* fEnabledByHost is @c true).
1089	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090	* @a fEnabledByHost is @c true).
1091	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092	* enable VT-x on the host.
1093	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1094	*/
1095	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096	void *pvMsrs)
1097	{
1098	Assert(pCpu);
1099	Assert(pvMsrs);
1100	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102	/* Enable VT-x if it's not already enabled by the host. */
1103	if (!fEnabledByHost)
1104	{
1105	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106	if (RT_FAILURE(rc))
1107	return rc;
1108	}
1109
1110	/*
1111	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1112	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1113	*/
1114	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1115	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1116	{
1117	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1118	pCpu->fFlushAsidBeforeUse = false;
1119	}
1120	else
1121	pCpu->fFlushAsidBeforeUse = true;
1122
1123	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1124	++pCpu->cTlbFlushes;
1125
1126	return VINF_SUCCESS;
1127	}
1128
1129
1130	/**
1131	* Deactivates VT-x on the current CPU.
1132	*
1133	* @returns VBox status code.
1134	* @param pCpu Pointer to the global CPU info struct.
1135	* @param pvCpuPage Pointer to the VMXON region.
1136	* @param HCPhysCpuPage Physical address of the VMXON region.
1137	*
1138	* @remarks This function should never be called when SUPR0EnableVTx() or
1139	* similar was used to enable VT-x on the host.
1140	*/
1141	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1142	{
1143	NOREF(pCpu);
1144	NOREF(pvCpuPage);
1145	NOREF(HCPhysCpuPage);
1146
1147	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1148	return hmR0VmxLeaveRootMode();
1149	}
1150
1151
1152	/**
1153	* Sets the permission bits for the specified MSR in the MSR bitmap.
1154	*
1155	* @param pVCpu The cross context virtual CPU structure.
1156	* @param uMsr The MSR value.
1157	* @param enmRead Whether reading this MSR causes a VM-exit.
1158	* @param enmWrite Whether writing this MSR causes a VM-exit.
1159	*/
1160	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1161	{
1162	int32_t iBit;
1163	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1164
1165	/*
1166	* Layout:
1167	* 0x000 - 0x3ff - Low MSR read bits
1168	* 0x400 - 0x7ff - High MSR read bits
1169	* 0x800 - 0xbff - Low MSR write bits
1170	* 0xc00 - 0xfff - High MSR write bits
1171	*/
1172	if (uMsr <= 0x00001FFF)
1173	iBit = uMsr;
1174	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1175	{
1176	iBit = uMsr - UINT32_C(0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1181
1182	Assert(iBit <= 0x1fff);
1183	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1184	ASMBitSet(pbMsrBitmap, iBit);
1185	else
1186	ASMBitClear(pbMsrBitmap, iBit);
1187
1188	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1189	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1190	else
1191	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1192	}
1193
1194
1195	#ifdef VBOX_STRICT
1196	/**
1197	* Gets the permission bits for the specified MSR in the MSR bitmap.
1198	*
1199	* @returns VBox status code.
1200	* @retval VINF_SUCCESS if the specified MSR is found.
1201	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1202	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1203	*
1204	* @param pVCpu The cross context virtual CPU structure.
1205	* @param uMsr The MSR.
1206	* @param penmRead Where to store the read permissions.
1207	* @param penmWrite Where to store the write permissions.
1208	*/
1209	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1210	{
1211	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1212	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1213	int32_t iBit;
1214	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1215
1216	/* See hmR0VmxSetMsrPermission() for the layout. */
1217	if (uMsr <= 0x00001FFF)
1218	iBit = uMsr;
1219	else if ( uMsr >= 0xC0000000
1220	&& uMsr <= 0xC0001FFF)
1221	{
1222	iBit = (uMsr - 0xC0000000);
1223	pbMsrBitmap += 0x400;
1224	}
1225	else
1226	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1227
1228	Assert(iBit <= 0x1fff);
1229	if (ASMBitTest(pbMsrBitmap, iBit))
1230	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1231	else
1232	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1233
1234	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1235	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1236	else
1237	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1238	return VINF_SUCCESS;
1239	}
1240	#endif /* VBOX_STRICT */
1241
1242
1243	/**
1244	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1245	* area.
1246	*
1247	* @returns VBox status code.
1248	* @param pVCpu The cross context virtual CPU structure.
1249	* @param cMsrs The number of MSRs.
1250	*/
1251	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1252	{
1253	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1254	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1255	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1256	{
1257	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1258	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1259	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1260	}
1261
1262	/* Update number of guest MSRs to load/store across the world-switch. */
1263	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1264	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1265
1266	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1267	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1268	AssertRCReturn(rc, rc);
1269
1270	/* Update the VCPU's copy of the MSR count. */
1271	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1272
1273	return VINF_SUCCESS;
1274	}
1275
1276
1277	/**
1278	* Adds a new (or updates the value of an existing) guest/host MSR
1279	* pair to be swapped during the world-switch as part of the
1280	* auto-load/store MSR area in the VMCS.
1281	*
1282	* @returns VBox status code.
1283	* @param pVCpu The cross context virtual CPU structure.
1284	* @param uMsr The MSR.
1285	* @param uGuestMsrValue Value of the guest MSR.
1286	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1287	* necessary.
1288	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1289	* its value was updated. Optional, can be NULL.
1290	*/
1291	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1292	bool *pfAddedAndUpdated)
1293	{
1294	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1295	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1296	uint32_t i;
1297	for (i = 0; i < cMsrs; i++)
1298	{
1299	if (pGuestMsr->u32Msr == uMsr)
1300	break;
1301	pGuestMsr++;
1302	}
1303
1304	bool fAdded = false;
1305	if (i == cMsrs)
1306	{
1307	++cMsrs;
1308	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1309	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1310
1311	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1312	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1313	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1314
1315	fAdded = true;
1316	}
1317
1318	/* Update the MSR values in the auto-load/store MSR area. */
1319	pGuestMsr->u32Msr = uMsr;
1320	pGuestMsr->u64Value = uGuestMsrValue;
1321
1322	/* Create/update the MSR slot in the host MSR area. */
1323	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1324	pHostMsr += i;
1325	pHostMsr->u32Msr = uMsr;
1326
1327	/*
1328	* Update the host MSR only when requested by the caller AND when we're
1329	* adding it to the auto-load/store area. Otherwise, it would have been
1330	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1331	*/
1332	bool fUpdatedMsrValue = false;
1333	if ( fAdded
1334	&& fUpdateHostMsr)
1335	{
1336	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1337	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1338	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1339	fUpdatedMsrValue = true;
1340	}
1341
1342	if (pfAddedAndUpdated)
1343	*pfAddedAndUpdated = fUpdatedMsrValue;
1344	return VINF_SUCCESS;
1345	}
1346
1347
1348	/**
1349	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1350	* auto-load/store MSR area in the VMCS.
1351	*
1352	* @returns VBox status code.
1353	* @param pVCpu The cross context virtual CPU structure.
1354	* @param uMsr The MSR.
1355	*/
1356	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1357	{
1358	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1359	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1360	for (uint32_t i = 0; i < cMsrs; i++)
1361	{
1362	/* Find the MSR. */
1363	if (pGuestMsr->u32Msr == uMsr)
1364	{
1365	/* If it's the last MSR, simply reduce the count. */
1366	if (i == cMsrs - 1)
1367	{
1368	--cMsrs;
1369	break;
1370	}
1371
1372	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1373	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1374	pLastGuestMsr += cMsrs - 1;
1375	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1376	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1377
1378	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380	pLastHostMsr += cMsrs - 1;
1381	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1382	pHostMsr->u64Value = pLastHostMsr->u64Value;
1383	--cMsrs;
1384	break;
1385	}
1386	pGuestMsr++;
1387	}
1388
1389	/* Update the VMCS if the count changed (meaning the MSR was found). */
1390	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1391	{
1392	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1393	AssertRCReturn(rc, rc);
1394
1395	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1396	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1397	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1398
1399	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1400	return VINF_SUCCESS;
1401	}
1402
1403	return VERR_NOT_FOUND;
1404	}
1405
1406
1407	/**
1408	* Checks if the specified guest MSR is part of the auto-load/store area in
1409	* the VMCS.
1410	*
1411	* @returns true if found, false otherwise.
1412	* @param pVCpu The cross context virtual CPU structure.
1413	* @param uMsr The MSR to find.
1414	*/
1415	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1416	{
1417	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1418	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1419
1420	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1421	{
1422	if (pGuestMsr->u32Msr == uMsr)
1423	return true;
1424	}
1425	return false;
1426	}
1427
1428
1429	/**
1430	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1431	*
1432	* @param pVCpu The cross context virtual CPU structure.
1433	*
1434	* @remarks No-long-jump zone!!!
1435	*/
1436	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1437	{
1438	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1439	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1440	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1441	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1442
1443	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1444	{
1445	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1446
1447	/*
1448	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1449	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1450	*/
1451	if (pHostMsr->u32Msr == MSR_K6_EFER)
1452	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1453	else
1454	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1455	}
1456
1457	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1458	}
1459
1460
1461	/**
1462	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1463	* perform lazy restoration of the host MSRs while leaving VT-x.
1464	*
1465	* @param pVCpu The cross context virtual CPU structure.
1466	*
1467	* @remarks No-long-jump zone!!!
1468	*/
1469	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1470	{
1471	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1472
1473	/*
1474	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1475	*/
1476	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477	{
1478	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1479	#if HC_ARCH_BITS == 64
1480	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1481	{
1482	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1483	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1484	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1485	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1486	}
1487	#endif
1488	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1489	}
1490	}
1491
1492
1493	/**
1494	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1495	* lazily while leaving VT-x.
1496	*
1497	* @returns true if it does, false otherwise.
1498	* @param pVCpu The cross context virtual CPU structure.
1499	* @param uMsr The MSR to check.
1500	*/
1501	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1502	{
1503	NOREF(pVCpu);
1504	#if HC_ARCH_BITS == 64
1505	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1506	{
1507	switch (uMsr)
1508	{
1509	case MSR_K8_LSTAR:
1510	case MSR_K6_STAR:
1511	case MSR_K8_SF_MASK:
1512	case MSR_K8_KERNEL_GS_BASE:
1513	return true;
1514	}
1515	}
1516	#else
1517	RT_NOREF(pVCpu, uMsr);
1518	#endif
1519	return false;
1520	}
1521
1522
1523	/**
1524	* Saves a set of guest MSRs back into the guest-CPU context.
1525	*
1526	* @param pVCpu The cross context virtual CPU structure.
1527	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1528	* out-of-sync. Make sure to update the required fields
1529	* before using them.
1530	*
1531	* @remarks No-long-jump zone!!!
1532	*/
1533	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1534	{
1535	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1536	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1537
1538	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1539	{
1540	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1541	#if HC_ARCH_BITS == 64
1542	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1543	{
1544	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1545	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1546	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1547	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1548	}
1549	#else
1550	NOREF(pMixedCtx);
1551	#endif
1552	}
1553	}
1554
1555
1556	/**
1557	* Loads a set of guests MSRs to allow read/passthru to the guest.
1558	*
1559	* The name of this function is slightly confusing. This function does NOT
1560	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1561	* common prefix for functions dealing with "lazy restoration" of the shared
1562	* MSRs.
1563	*
1564	* @param pVCpu The cross context virtual CPU structure.
1565	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1566	* out-of-sync. Make sure to update the required fields
1567	* before using them.
1568	*
1569	* @remarks No-long-jump zone!!!
1570	*/
1571	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1572	{
1573	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1574	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1575
1576	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1577	#if HC_ARCH_BITS == 64
1578	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1579	{
1580	/*
1581	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1582	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1583	* we can skip a few MSR writes.
1584	*
1585	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1586	* guest MSR values in the guest-CPU context might be different to what's currently
1587	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1588	* CPU, see @bugref{8728}.
1589	*/
1590	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1591	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1592	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1593	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1594	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1595	{
1596	#ifdef VBOX_STRICT
1597	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1598	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1599	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1600	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1601	#endif
1602	}
1603	else
1604	{
1605	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1606	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1607	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1608	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1609	}
1610	}
1611	#else
1612	RT_NOREF(pMixedCtx);
1613	#endif
1614	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1615	}
1616
1617
1618	/**
1619	* Performs lazy restoration of the set of host MSRs if they were previously
1620	* loaded with guest MSR values.
1621	*
1622	* @param pVCpu The cross context virtual CPU structure.
1623	*
1624	* @remarks No-long-jump zone!!!
1625	* @remarks The guest MSRs should have been saved back into the guest-CPU
1626	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1627	*/
1628	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1629	{
1630	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1631	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1632
1633	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1634	{
1635	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1636	#if HC_ARCH_BITS == 64
1637	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1638	{
1639	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1640	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1641	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1642	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1643	}
1644	#endif
1645	}
1646	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1647	}
1648
1649
1650	/**
1651	* Verifies that our cached values of the VMCS controls are all
1652	* consistent with what's actually present in the VMCS.
1653	*
1654	* @returns VBox status code.
1655	* @param pVCpu The cross context virtual CPU structure.
1656	*/
1657	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1658	{
1659	uint32_t u32Val;
1660	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1661	AssertRCReturn(rc, rc);
1662	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1663	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1664
1665	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1666	AssertRCReturn(rc, rc);
1667	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1668	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1669
1670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1671	AssertRCReturn(rc, rc);
1672	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1673	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1674
1675	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1676	AssertRCReturn(rc, rc);
1677	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1678	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1679
1680	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1681	{
1682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1683	AssertRCReturn(rc, rc);
1684	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1685	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1686	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1687	}
1688
1689	return VINF_SUCCESS;
1690	}
1691
1692
1693	#ifdef VBOX_STRICT
1694	/**
1695	* Verifies that our cached host EFER value has not changed
1696	* since we cached it.
1697	*
1698	* @param pVCpu The cross context virtual CPU structure.
1699	*/
1700	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1701	{
1702	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1703
1704	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1705	{
1706	uint64_t u64Val;
1707	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1708	AssertRC(rc);
1709
1710	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1711	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1712	}
1713	}
1714
1715
1716	/**
1717	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1718	* VMCS are correct.
1719	*
1720	* @param pVCpu The cross context virtual CPU structure.
1721	*/
1722	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1723	{
1724	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1725
1726	/* Verify MSR counts in the VMCS are what we think it should be. */
1727	uint32_t cMsrs;
1728	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1729	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1730
1731	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1732	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1733
1734	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1735	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1736
1737	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1738	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1739	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1740	{
1741	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1742	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1743	pGuestMsr->u32Msr, cMsrs));
1744
1745	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1746	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1747	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1748
1749	/* Verify that the permissions are as expected in the MSR bitmap. */
1750	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1751	{
1752	VMXMSREXITREAD enmRead;
1753	VMXMSREXITWRITE enmWrite;
1754	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1755	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1756	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1757	{
1758	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1759	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1760	}
1761	else
1762	{
1763	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1764	pGuestMsr->u32Msr, cMsrs));
1765	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1766	pGuestMsr->u32Msr, cMsrs));
1767	}
1768	}
1769	}
1770	}
1771	#endif /* VBOX_STRICT */
1772
1773
1774	/**
1775	* Flushes the TLB using EPT.
1776	*
1777	* @returns VBox status code.
1778	* @param pVCpu The cross context virtual CPU structure of the calling
1779	* EMT. Can be NULL depending on @a enmFlush.
1780	* @param enmFlush Type of flush.
1781	*
1782	* @remarks Caller is responsible for making sure this function is called only
1783	* when NestedPaging is supported and providing @a enmFlush that is
1784	* supported by the CPU.
1785	* @remarks Can be called with interrupts disabled.
1786	*/
1787	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1788	{
1789	uint64_t au64Descriptor[2];
1790	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1791	au64Descriptor[0] = 0;
1792	else
1793	{
1794	Assert(pVCpu);
1795	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1796	}
1797	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1798
1799	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1800	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1801	rc));
1802	if ( RT_SUCCESS(rc)
1803	&& pVCpu)
1804	{
1805	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1806	}
1807	}
1808
1809
1810	/**
1811	* Flushes the TLB using VPID.
1812	*
1813	* @returns VBox status code.
1814	* @param pVM The cross context VM structure.
1815	* @param pVCpu The cross context virtual CPU structure of the calling
1816	* EMT. Can be NULL depending on @a enmFlush.
1817	* @param enmFlush Type of flush.
1818	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1819	* on @a enmFlush).
1820	*
1821	* @remarks Can be called with interrupts disabled.
1822	*/
1823	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1824	{
1825	NOREF(pVM);
1826	AssertPtr(pVM);
1827	Assert(pVM->hm.s.vmx.fVpid);
1828
1829	uint64_t au64Descriptor[2];
1830	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1831	{
1832	au64Descriptor[0] = 0;
1833	au64Descriptor[1] = 0;
1834	}
1835	else
1836	{
1837	AssertPtr(pVCpu);
1838	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1839	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1840	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1841	au64Descriptor[1] = GCPtr;
1842	}
1843
1844	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1845	AssertMsg(rc == VINF_SUCCESS,
1846	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1847	if ( RT_SUCCESS(rc)
1848	&& pVCpu)
1849	{
1850	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1851	}
1852	}
1853
1854
1855	/**
1856	* Invalidates a guest page by guest virtual address. Only relevant for
1857	* EPT/VPID, otherwise there is nothing really to invalidate.
1858	*
1859	* @returns VBox status code.
1860	* @param pVM The cross context VM structure.
1861	* @param pVCpu The cross context virtual CPU structure.
1862	* @param GCVirt Guest virtual address of the page to invalidate.
1863	*/
1864	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1865	{
1866	AssertPtr(pVM);
1867	AssertPtr(pVCpu);
1868	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1869
1870	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1871	if (!fFlushPending)
1872	{
1873	/*
1874	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1875	* See @bugref{6043} and @bugref{6177}.
1876	*
1877	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1878	* function maybe called in a loop with individual addresses.
1879	*/
1880	if (pVM->hm.s.vmx.fVpid)
1881	{
1882	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1883	{
1884	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1885	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1886	}
1887	else
1888	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889	}
1890	else if (pVM->hm.s.fNestedPaging)
1891	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1892	}
1893
1894	return VINF_SUCCESS;
1895	}
1896
1897
1898	/**
1899	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1900	* otherwise there is nothing really to invalidate.
1901	*
1902	* @returns VBox status code.
1903	* @param pVM The cross context VM structure.
1904	* @param pVCpu The cross context virtual CPU structure.
1905	* @param GCPhys Guest physical address of the page to invalidate.
1906	*/
1907	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1908	{
1909	NOREF(pVM); NOREF(GCPhys);
1910	LogFlowFunc(("%RGp\n", GCPhys));
1911
1912	/*
1913	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1914	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1915	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1916	*/
1917	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1918	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1919	return VINF_SUCCESS;
1920	}
1921
1922
1923	/**
1924	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1925	* case where neither EPT nor VPID is supported by the CPU.
1926	*
1927	* @param pVM The cross context VM structure.
1928	* @param pVCpu The cross context virtual CPU structure.
1929	* @param pCpu Pointer to the global HM struct.
1930	*
1931	* @remarks Called with interrupts disabled.
1932	*/
1933	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1934	{
1935	AssertPtr(pVCpu);
1936	AssertPtr(pCpu);
1937	NOREF(pVM);
1938
1939	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1940
1941	Assert(pCpu->idCpu != NIL_RTCPUID);
1942	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1943	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1944	pVCpu->hm.s.fForceTLBFlush = false;
1945	return;
1946	}
1947
1948
1949	/**
1950	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1951	*
1952	* @param pVM The cross context VM structure.
1953	* @param pVCpu The cross context virtual CPU structure.
1954	* @param pCpu Pointer to the global HM CPU struct.
1955	* @remarks All references to "ASID" in this function pertains to "VPID" in
1956	* Intel's nomenclature. The reason is, to avoid confusion in compare
1957	* statements since the host-CPU copies are named "ASID".
1958	*
1959	* @remarks Called with interrupts disabled.
1960	*/
1961	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1962	{
1963	#ifdef VBOX_WITH_STATISTICS
1964	bool fTlbFlushed = false;
1965	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1966	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1967	if (!fTlbFlushed) \
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1969	} while (0)
1970	#else
1971	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1972	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1973	#endif
1974
1975	AssertPtr(pVM);
1976	AssertPtr(pCpu);
1977	AssertPtr(pVCpu);
1978	Assert(pCpu->idCpu != NIL_RTCPUID);
1979
1980	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1981	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1982	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1983
1984	/*
1985	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1986	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1987	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1988	*/
1989	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1990	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1991	{
1992	++pCpu->uCurrentAsid;
1993	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1994	{
1995	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1996	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1997	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1998	}
1999
2000	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2001	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2002	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2003
2004	/*
2005	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2006	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2007	*/
2008	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2009	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2010	HMVMX_SET_TAGGED_TLB_FLUSHED();
2011	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2012	}
2013
2014	/* Check for explicit TLB flushes. */
2015	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2016	{
2017	/*
2018	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2019	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2020	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2021	* but not guest-physical mappings.
2022	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2023	*/
2024	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2025	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2026	HMVMX_SET_TAGGED_TLB_FLUSHED();
2027	}
2028
2029	pVCpu->hm.s.fForceTLBFlush = false;
2030	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2031
2032	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2033	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2034	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2035	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2036	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2037	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2038	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2039	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2040	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2041
2042	/* Update VMCS with the VPID. */
2043	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2044	AssertRC(rc);
2045
2046	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2047	}
2048
2049
2050	/**
2051	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2052	*
2053	* @returns VBox status code.
2054	* @param pVM The cross context VM structure.
2055	* @param pVCpu The cross context virtual CPU structure.
2056	* @param pCpu Pointer to the global HM CPU struct.
2057	*
2058	* @remarks Called with interrupts disabled.
2059	*/
2060	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2061	{
2062	AssertPtr(pVM);
2063	AssertPtr(pVCpu);
2064	AssertPtr(pCpu);
2065	Assert(pCpu->idCpu != NIL_RTCPUID);
2066	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2067	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2068
2069	/*
2070	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2071	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2072	*/
2073	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2074	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2075	{
2076	pVCpu->hm.s.fForceTLBFlush = true;
2077	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2078	}
2079
2080	/* Check for explicit TLB flushes. */
2081	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2082	{
2083	pVCpu->hm.s.fForceTLBFlush = true;
2084	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2085	}
2086
2087	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2088	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2089
2090	if (pVCpu->hm.s.fForceTLBFlush)
2091	{
2092	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2093	pVCpu->hm.s.fForceTLBFlush = false;
2094	}
2095	}
2096
2097
2098	/**
2099	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2100	*
2101	* @returns VBox status code.
2102	* @param pVM The cross context VM structure.
2103	* @param pVCpu The cross context virtual CPU structure.
2104	* @param pCpu Pointer to the global HM CPU struct.
2105	*
2106	* @remarks Called with interrupts disabled.
2107	*/
2108	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2109	{
2110	AssertPtr(pVM);
2111	AssertPtr(pVCpu);
2112	AssertPtr(pCpu);
2113	Assert(pCpu->idCpu != NIL_RTCPUID);
2114	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2115	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2116
2117	/*
2118	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2119	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2120	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2121	*/
2122	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2123	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2124	{
2125	pVCpu->hm.s.fForceTLBFlush = true;
2126	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2127	}
2128
2129	/* Check for explicit TLB flushes. */
2130	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2131	{
2132	/*
2133	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2134	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2135	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2136	*/
2137	pVCpu->hm.s.fForceTLBFlush = true;
2138	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2139	}
2140
2141	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2142	if (pVCpu->hm.s.fForceTLBFlush)
2143	{
2144	++pCpu->uCurrentAsid;
2145	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2146	{
2147	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2148	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2149	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2150	}
2151
2152	pVCpu->hm.s.fForceTLBFlush = false;
2153	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2154	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2155	if (pCpu->fFlushAsidBeforeUse)
2156	{
2157	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2158	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2159	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2160	{
2161	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2162	pCpu->fFlushAsidBeforeUse = false;
2163	}
2164	else
2165	{
2166	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2167	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2168	}
2169	}
2170	}
2171
2172	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2173	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2174	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2175	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2176	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2177	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2178	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2179
2180	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2181	AssertRC(rc);
2182	}
2183
2184
2185	/**
2186	* Flushes the guest TLB entry based on CPU capabilities.
2187	*
2188	* @param pVCpu The cross context virtual CPU structure.
2189	* @param pCpu Pointer to the global HM CPU struct.
2190	*/
2191	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2192	{
2193	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2194	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2195	#endif
2196	PVM pVM = pVCpu->CTX_SUFF(pVM);
2197	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2198	{
2199	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2200	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2201	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2202	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2203	default:
2204	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2205	break;
2206	}
2207
2208	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2209	}
2210
2211
2212	/**
2213	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2214	* TLB entries from the host TLB before VM-entry.
2215	*
2216	* @returns VBox status code.
2217	* @param pVM The cross context VM structure.
2218	*/
2219	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2220	{
2221	/*
2222	* Determine optimal flush type for Nested Paging.
2223	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2224	* guest execution (see hmR3InitFinalizeR0()).
2225	*/
2226	if (pVM->hm.s.fNestedPaging)
2227	{
2228	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2229	{
2230	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2231	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2232	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2233	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2234	else
2235	{
2236	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2237	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2238	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2239	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2240	}
2241
2242	/* Make sure the write-back cacheable memory type for EPT is supported. */
2243	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2244	{
2245	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2246	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2247	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2248	}
2249
2250	/* EPT requires a page-walk length of 4. */
2251	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2252	{
2253	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2254	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2255	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2256	}
2257	}
2258	else
2259	{
2260	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2261	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2263	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264	}
2265	}
2266
2267	/*
2268	* Determine optimal flush type for VPID.
2269	*/
2270	if (pVM->hm.s.vmx.fVpid)
2271	{
2272	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2273	{
2274	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2275	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2276	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2277	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2278	else
2279	{
2280	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2281	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2282	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2283	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2284	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2285	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2286	pVM->hm.s.vmx.fVpid = false;
2287	}
2288	}
2289	else
2290	{
2291	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2292	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2293	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2294	pVM->hm.s.vmx.fVpid = false;
2295	}
2296	}
2297
2298	/*
2299	* Setup the handler for flushing tagged-TLBs.
2300	*/
2301	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2302	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2303	else if (pVM->hm.s.fNestedPaging)
2304	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2305	else if (pVM->hm.s.vmx.fVpid)
2306	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2307	else
2308	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2309	return VINF_SUCCESS;
2310	}
2311
2312
2313	/**
2314	* Sets up pin-based VM-execution controls in the VMCS.
2315	*
2316	* @returns VBox status code.
2317	* @param pVM The cross context VM structure.
2318	* @param pVCpu The cross context virtual CPU structure.
2319	*/
2320	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2321	{
2322	AssertPtr(pVM);
2323	AssertPtr(pVCpu);
2324
2325	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2326	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2327
2328	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2329	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2330
2331	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2332	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2333
2334	/* Enable the VMX preemption timer. */
2335	if (pVM->hm.s.vmx.fUsePreemptTimer)
2336	{
2337	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2338	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2339	}
2340
2341	#if 0
2342	/* Enable posted-interrupt processing. */
2343	if (pVM->hm.s.fPostedIntrs)
2344	{
2345	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2346	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2347	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2348	}
2349	#endif
2350
2351	if ((val & zap) != val)
2352	{
2353	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2354	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2355	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2356	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2357	}
2358
2359	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2360	AssertRCReturn(rc, rc);
2361
2362	pVCpu->hm.s.vmx.u32PinCtls = val;
2363	return rc;
2364	}
2365
2366
2367	/**
2368	* Sets up processor-based VM-execution controls in the VMCS.
2369	*
2370	* @returns VBox status code.
2371	* @param pVM The cross context VM structure.
2372	* @param pVCpu The cross context virtual CPU structure.
2373	*/
2374	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2375	{
2376	AssertPtr(pVM);
2377	AssertPtr(pVCpu);
2378
2379	int rc = VERR_INTERNAL_ERROR_5;
2380	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2381	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2382
2383	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2384	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2385	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2386	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2387	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2388	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2389	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2390
2391	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2392	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2393	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2394	{
2395	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2396	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2397	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2398	}
2399
2400	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2401	if (!pVM->hm.s.fNestedPaging)
2402	{
2403	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2404	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2405	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2406	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2407	}
2408
2409	/* Use TPR shadowing if supported by the CPU. */
2410	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2411	{
2412	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2413	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2414	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2415	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2416	AssertRCReturn(rc, rc);
2417
2418	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2419	/* CR8 writes cause a VM-exit based on TPR threshold. */
2420	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2421	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2422	}
2423	else
2424	{
2425	/*
2426	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2427	* Set this control only for 64-bit guests.
2428	*/
2429	if (pVM->hm.s.fAllow64BitGuests)
2430	{
2431	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2432	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2433	}
2434	}
2435
2436	/* Use MSR-bitmaps if supported by the CPU. */
2437	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2438	{
2439	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2440
2441	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2442	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2443	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2444	AssertRCReturn(rc, rc);
2445
2446	/*
2447	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2448	* automatically using dedicated fields in the VMCS.
2449	*/
2450	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2455
2456	#if HC_ARCH_BITS == 64
2457	/*
2458	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2459	*/
2460	if (pVM->hm.s.fAllow64BitGuests)
2461	{
2462	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2463	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2464	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2465	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2466	}
2467	#endif
2468	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2469	}
2470
2471	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2472	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2473	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2474
2475	if ((val & zap) != val)
2476	{
2477	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2478	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2479	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2480	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2481	}
2482
2483	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2484	AssertRCReturn(rc, rc);
2485
2486	pVCpu->hm.s.vmx.u32ProcCtls = val;
2487
2488	/*
2489	* Secondary processor-based VM-execution controls.
2490	*/
2491	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2492	{
2493	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2494	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2495
2496	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2497	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2498
2499	if (pVM->hm.s.fNestedPaging)
2500	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2501	else
2502	{
2503	/*
2504	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2505	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2506	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2507	*/
2508	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2509	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2510	}
2511
2512	if (pVM->hm.s.vmx.fVpid)
2513	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2514
2515	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2516	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2517
2518	#if 0
2519	if (pVM->hm.s.fVirtApicRegs)
2520	{
2521	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2522	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2523
2524	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2525	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2526	}
2527	#endif
2528
2529	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2530	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2531	* done dynamically. */
2532	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2533	{
2534	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2535	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2536	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2537	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2538	AssertRCReturn(rc, rc);
2539	}
2540
2541	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2542	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2543
2544	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2545	&& pVM->hm.s.vmx.cPleGapTicks
2546	&& pVM->hm.s.vmx.cPleWindowTicks)
2547	{
2548	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2549
2550	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2551	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2552	AssertRCReturn(rc, rc);
2553	}
2554
2555	if ((val & zap) != val)
2556	{
2557	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2558	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2559	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2560	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2561	}
2562
2563	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2564	AssertRCReturn(rc, rc);
2565
2566	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2567	}
2568	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2569	{
2570	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2571	"available\n"));
2572	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2573	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2574	}
2575
2576	return VINF_SUCCESS;
2577	}
2578
2579
2580	/**
2581	* Sets up miscellaneous (everything other than Pin & Processor-based
2582	* VM-execution) control fields in the VMCS.
2583	*
2584	* @returns VBox status code.
2585	* @param pVM The cross context VM structure.
2586	* @param pVCpu The cross context virtual CPU structure.
2587	*/
2588	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2589	{
2590	NOREF(pVM);
2591	AssertPtr(pVM);
2592	AssertPtr(pVCpu);
2593
2594	int rc = VERR_GENERAL_FAILURE;
2595
2596	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2597	#if 0
2598	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2599	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2600	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2601
2602	/*
2603	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2604	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2605	* We thus use the exception bitmap to control it rather than use both.
2606	*/
2607	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2608	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2609
2610	/** @todo Explore possibility of using IO-bitmaps. */
2611	/* All IO & IOIO instructions cause VM-exits. */
2612	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2613	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2614
2615	/* Initialize the MSR-bitmap area. */
2616	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2617	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2618	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2619	AssertRCReturn(rc, rc);
2620	#endif
2621
2622	/* Setup MSR auto-load/store area. */
2623	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2624	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2625	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2626	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2627	AssertRCReturn(rc, rc);
2628
2629	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2630	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2631	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2632	AssertRCReturn(rc, rc);
2633
2634	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2635	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2636	AssertRCReturn(rc, rc);
2637
2638	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2639	#if 0
2640	/* Setup debug controls */
2641	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2642	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2643	AssertRCReturn(rc, rc);
2644	#endif
2645
2646	return rc;
2647	}
2648
2649
2650	/**
2651	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2652	*
2653	* We shall setup those exception intercepts that don't change during the
2654	* lifetime of the VM here. The rest are done dynamically while loading the
2655	* guest state.
2656	*
2657	* @returns VBox status code.
2658	* @param pVM The cross context VM structure.
2659	* @param pVCpu The cross context virtual CPU structure.
2660	*/
2661	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2662	{
2663	AssertPtr(pVM);
2664	AssertPtr(pVCpu);
2665
2666	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2667
2668	uint32_t u32XcptBitmap = 0;
2669
2670	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2671	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2672
2673	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2674	and writes, and because recursive #DBs can cause the CPU hang, we must always
2675	intercept #DB. */
2676	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2677
2678	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2679	if (!pVM->hm.s.fNestedPaging)
2680	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2681
2682	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2683	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2684	AssertRCReturn(rc, rc);
2685	return rc;
2686	}
2687
2688
2689	/**
2690	* Sets up the initial guest-state mask. The guest-state mask is consulted
2691	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2692	* for the nested virtualization case (as it would cause a VM-exit).
2693	*
2694	* @param pVCpu The cross context virtual CPU structure.
2695	*/
2696	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2697	{
2698	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2699	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2700	return VINF_SUCCESS;
2701	}
2702
2703
2704	/**
2705	* Does per-VM VT-x initialization.
2706	*
2707	* @returns VBox status code.
2708	* @param pVM The cross context VM structure.
2709	*/
2710	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2711	{
2712	LogFlowFunc(("pVM=%p\n", pVM));
2713
2714	int rc = hmR0VmxStructsAlloc(pVM);
2715	if (RT_FAILURE(rc))
2716	{
2717	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2718	return rc;
2719	}
2720
2721	return VINF_SUCCESS;
2722	}
2723
2724
2725	/**
2726	* Does per-VM VT-x termination.
2727	*
2728	* @returns VBox status code.
2729	* @param pVM The cross context VM structure.
2730	*/
2731	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2732	{
2733	LogFlowFunc(("pVM=%p\n", pVM));
2734
2735	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2736	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2737	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2738	#endif
2739	hmR0VmxStructsFree(pVM);
2740	return VINF_SUCCESS;
2741	}
2742
2743
2744	/**
2745	* Sets up the VM for execution under VT-x.
2746	* This function is only called once per-VM during initialization.
2747	*
2748	* @returns VBox status code.
2749	* @param pVM The cross context VM structure.
2750	*/
2751	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2752	{
2753	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2754	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2755
2756	LogFlowFunc(("pVM=%p\n", pVM));
2757
2758	/*
2759	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2760	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2761	*/
2762	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2763	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2764	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2765	{
2766	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2767	return VERR_INTERNAL_ERROR;
2768	}
2769
2770	/* Initialize these always, see hmR3InitFinalizeR0().*/
2771	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2772	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2773
2774	/* Setup the tagged-TLB flush handlers. */
2775	int rc = hmR0VmxSetupTaggedTlb(pVM);
2776	if (RT_FAILURE(rc))
2777	{
2778	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2779	return rc;
2780	}
2781
2782	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2783	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2784	#if HC_ARCH_BITS == 64
2785	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2786	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2787	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2788	{
2789	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2790	}
2791	#endif
2792
2793	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2794	RTCCUINTREG uHostCR4 = ASMGetCR4();
2795	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2796	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2797
2798	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2799	{
2800	PVMCPU pVCpu = &pVM->aCpus[i];
2801	AssertPtr(pVCpu);
2802	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2803
2804	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2805	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2806
2807	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2808	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2809	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2810
2811	/* Set revision dword at the beginning of the VMCS structure. */
2812	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2813
2814	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2815	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2816	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818
2819	/* Load this VMCS as the current VMCS. */
2820	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2821	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2825	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2826	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2827
2828	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2829	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2830	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2831
2832	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2833	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2834	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2835
2836	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2837	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2838	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2839
2840	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2841	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2842	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2843
2844	#if HC_ARCH_BITS == 32
2845	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2846	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2847	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2848	#endif
2849
2850	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2851	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2852	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2853	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2854
2855	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2856
2857	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2858	}
2859
2860	return VINF_SUCCESS;
2861	}
2862
2863
2864	/**
2865	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2866	* the VMCS.
2867	*
2868	* @returns VBox status code.
2869	* @param pVM The cross context VM structure.
2870	* @param pVCpu The cross context virtual CPU structure.
2871	*/
2872	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2873	{
2874	NOREF(pVM); NOREF(pVCpu);
2875
2876	RTCCUINTREG uReg = ASMGetCR0();
2877	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2878	AssertRCReturn(rc, rc);
2879
2880	uReg = ASMGetCR3();
2881	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2882	AssertRCReturn(rc, rc);
2883
2884	uReg = ASMGetCR4();
2885	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2886	AssertRCReturn(rc, rc);
2887	return rc;
2888	}
2889
2890
2891	#if HC_ARCH_BITS == 64
2892	/**
2893	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2894	* requirements. See hmR0VmxSaveHostSegmentRegs().
2895	*/
2896	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2897	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2898	{ \
2899	bool fValidSelector = true; \
2900	if ((selValue) & X86_SEL_LDT) \
2901	{ \
2902	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2903	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2904	} \
2905	if (fValidSelector) \
2906	{ \
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2908	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2909	} \
2910	(selValue) = 0; \
2911	}
2912	#endif
2913
2914
2915	/**
2916	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2917	* the host-state area in the VMCS.
2918	*
2919	* @returns VBox status code.
2920	* @param pVM The cross context VM structure.
2921	* @param pVCpu The cross context virtual CPU structure.
2922	*/
2923	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2924	{
2925	int rc = VERR_INTERNAL_ERROR_5;
2926
2927	#if HC_ARCH_BITS == 64
2928	/*
2929	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2930	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2931	*
2932	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2933	* Was observed booting Solaris10u10 32-bit guest.
2934	*/
2935	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2936	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2937	{
2938	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2939	pVCpu->idCpu));
2940	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2941	}
2942	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2943	#else
2944	RT_NOREF(pVCpu);
2945	#endif
2946
2947	/*
2948	* Host DS, ES, FS and GS segment registers.
2949	*/
2950	#if HC_ARCH_BITS == 64
2951	RTSEL uSelDS = ASMGetDS();
2952	RTSEL uSelES = ASMGetES();
2953	RTSEL uSelFS = ASMGetFS();
2954	RTSEL uSelGS = ASMGetGS();
2955	#else
2956	RTSEL uSelDS = 0;
2957	RTSEL uSelES = 0;
2958	RTSEL uSelFS = 0;
2959	RTSEL uSelGS = 0;
2960	#endif
2961
2962	/*
2963	* Host CS and SS segment registers.
2964	*/
2965	RTSEL uSelCS = ASMGetCS();
2966	RTSEL uSelSS = ASMGetSS();
2967
2968	/*
2969	* Host TR segment register.
2970	*/
2971	RTSEL uSelTR = ASMGetTR();
2972
2973	#if HC_ARCH_BITS == 64
2974	/*
2975	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2976	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2977	*/
2978	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2979	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2980	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2981	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2982	# undef VMXLOCAL_ADJUST_HOST_SEG
2983	#endif
2984
2985	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2986	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2987	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2988	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2989	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2990	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2991	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2992	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2993	Assert(uSelCS);
2994	Assert(uSelTR);
2995
2996	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2997	#if 0
2998	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2999	Assert(uSelSS != 0);
3000	#endif
3001
3002	/* Write these host selector fields into the host-state area in the VMCS. */
3003	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3004	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3005	#if HC_ARCH_BITS == 64
3006	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3007	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3008	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3009	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3010	#else
3011	NOREF(uSelDS);
3012	NOREF(uSelES);
3013	NOREF(uSelFS);
3014	NOREF(uSelGS);
3015	#endif
3016	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3017	AssertRCReturn(rc, rc);
3018
3019	/*
3020	* Host GDTR and IDTR.
3021	*/
3022	RTGDTR Gdtr;
3023	RTIDTR Idtr;
3024	RT_ZERO(Gdtr);
3025	RT_ZERO(Idtr);
3026	ASMGetGDTR(&Gdtr);
3027	ASMGetIDTR(&Idtr);
3028	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3029	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3030	AssertRCReturn(rc, rc);
3031
3032	#if HC_ARCH_BITS == 64
3033	/*
3034	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3035	* maximum limit (0xffff) on every VM-exit.
3036	*/
3037	if (Gdtr.cbGdt != 0xffff)
3038	{
3039	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3040	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3041	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3042	}
3043
3044	/*
3045	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3046	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3047	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3048	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3049	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3050	* hosts where we are pretty sure it won't cause trouble.
3051	*/
3052	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3053	if (Idtr.cbIdt < 0x0fff)
3054	# else
3055	if (Idtr.cbIdt != 0xffff)
3056	# endif
3057	{
3058	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3059	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3060	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3061	}
3062	#endif
3063
3064	/*
3065	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3066	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3067	*/
3068	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3069	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3070	VERR_VMX_INVALID_HOST_STATE);
3071
3072	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3073	#if HC_ARCH_BITS == 64
3074	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3075
3076	/*
3077	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3078	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3079	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3080	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3081	*
3082	* [1] See Intel spec. 3.5 "System Descriptor Types".
3083	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3084	*/
3085	Assert(pDesc->System.u4Type == 11);
3086	if ( pDesc->System.u16LimitLow != 0x67
3087	\|\| pDesc->System.u4LimitHigh)
3088	{
3089	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3090	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3091	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3092	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3093	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3094
3095	/* Store the GDTR here as we need it while restoring TR. */
3096	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3097	}
3098	#else
3099	NOREF(pVM);
3100	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3101	#endif
3102	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3103	AssertRCReturn(rc, rc);
3104
3105	/*
3106	* Host FS base and GS base.
3107	*/
3108	#if HC_ARCH_BITS == 64
3109	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3110	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3111	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3112	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3113	AssertRCReturn(rc, rc);
3114
3115	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3116	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3117	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3118	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3119	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3120	#endif
3121	return rc;
3122	}
3123
3124
3125	/**
3126	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3127	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3128	* the host after every successful VM-exit.
3129	*
3130	* @returns VBox status code.
3131	* @param pVM The cross context VM structure.
3132	* @param pVCpu The cross context virtual CPU structure.
3133	*
3134	* @remarks No-long-jump zone!!!
3135	*/
3136	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3137	{
3138	NOREF(pVM);
3139
3140	AssertPtr(pVCpu);
3141	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3142
3143	/*
3144	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3145	* rather than swapping them on every VM-entry.
3146	*/
3147	hmR0VmxLazySaveHostMsrs(pVCpu);
3148
3149	/*
3150	* Host Sysenter MSRs.
3151	*/
3152	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3153	#if HC_ARCH_BITS == 32
3154	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3155	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3156	#else
3157	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3158	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3159	#endif
3160	AssertRCReturn(rc, rc);
3161
3162	/*
3163	* Host EFER MSR.
3164	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3165	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3166	*/
3167	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3168	{
3169	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3170	AssertRCReturn(rc, rc);
3171	}
3172
3173	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3174	* hmR0VmxLoadGuestExitCtls() !! */
3175
3176	return rc;
3177	}
3178
3179
3180	/**
3181	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3182	*
3183	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3184	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3185	* hmR0VMxLoadGuestEntryCtls().
3186	*
3187	* @returns true if we need to load guest EFER, false otherwise.
3188	* @param pVCpu The cross context virtual CPU structure.
3189	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3190	* out-of-sync. Make sure to update the required fields
3191	* before using them.
3192	*
3193	* @remarks Requires EFER, CR4.
3194	* @remarks No-long-jump zone!!!
3195	*/
3196	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3197	{
3198	#ifdef HMVMX_ALWAYS_SWAP_EFER
3199	return true;
3200	#endif
3201
3202	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3203	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3204	if (CPUMIsGuestInLongMode(pVCpu))
3205	return false;
3206	#endif
3207
3208	PVM pVM = pVCpu->CTX_SUFF(pVM);
3209	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3210	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3211
3212	/*
3213	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3214	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3215	*/
3216	if ( CPUMIsGuestInLongMode(pVCpu)
3217	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3218	{
3219	return true;
3220	}
3221
3222	/*
3223	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3224	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3225	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3226	*/
3227	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3228	&& (pMixedCtx->cr0 & X86_CR0_PG)
3229	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3230	{
3231	/* Assert that host is PAE capable. */
3232	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3233	return true;
3234	}
3235
3236	/** @todo Check the latest Intel spec. for any other bits,
3237	* like SMEP/SMAP? */
3238	return false;
3239	}
3240
3241
3242	/**
3243	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3244	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3245	* controls".
3246	*
3247	* @returns VBox status code.
3248	* @param pVCpu The cross context virtual CPU structure.
3249	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3250	* out-of-sync. Make sure to update the required fields
3251	* before using them.
3252	*
3253	* @remarks Requires EFER.
3254	* @remarks No-long-jump zone!!!
3255	*/
3256	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3257	{
3258	int rc = VINF_SUCCESS;
3259	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3260	{
3261	PVM pVM = pVCpu->CTX_SUFF(pVM);
3262	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3263	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3264
3265	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3266	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3267
3268	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3269	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3270	{
3271	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3272	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3273	}
3274	else
3275	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3276
3277	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3278	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3279	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3280	{
3281	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3282	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3283	}
3284
3285	/*
3286	* The following should -not- be set (since we're not in SMM mode):
3287	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3288	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3289	*/
3290
3291	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3292	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3293
3294	if ((val & zap) != val)
3295	{
3296	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3297	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3298	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3299	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3300	}
3301
3302	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3303	AssertRCReturn(rc, rc);
3304
3305	pVCpu->hm.s.vmx.u32EntryCtls = val;
3306	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3307	}
3308	return rc;
3309	}
3310
3311
3312	/**
3313	* Sets up the VM-exit controls in the VMCS.
3314	*
3315	* @returns VBox status code.
3316	* @param pVCpu The cross context virtual CPU structure.
3317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3318	* out-of-sync. Make sure to update the required fields
3319	* before using them.
3320	*
3321	* @remarks Requires EFER.
3322	*/
3323	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3324	{
3325	NOREF(pMixedCtx);
3326
3327	int rc = VINF_SUCCESS;
3328	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3329	{
3330	PVM pVM = pVCpu->CTX_SUFF(pVM);
3331	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3332	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3333
3334	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3335	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3336
3337	/*
3338	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3339	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3340	*/
3341	#if HC_ARCH_BITS == 64
3342	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3343	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3344	#else
3345	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3346	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3347	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3348	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3349	{
3350	/* The switcher returns to long mode, EFER is managed by the switcher. */
3351	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3352	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3353	}
3354	else
3355	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3356	#endif
3357
3358	/* If the newer VMCS fields for managing EFER exists, use it. */
3359	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3360	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3361	{
3362	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3363	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3364	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3365	}
3366
3367	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3368	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3369
3370	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3371	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3372	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3373
3374	if ( pVM->hm.s.vmx.fUsePreemptTimer
3375	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3376	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3377
3378	if ((val & zap) != val)
3379	{
3380	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3381	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3382	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3383	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3384	}
3385
3386	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3387	AssertRCReturn(rc, rc);
3388
3389	pVCpu->hm.s.vmx.u32ExitCtls = val;
3390	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3391	}
3392	return rc;
3393	}
3394
3395
3396	/**
3397	* Sets the TPR threshold in the VMCS.
3398	*
3399	* @returns VBox status code.
3400	* @param pVCpu The cross context virtual CPU structure.
3401	* @param u32TprThreshold The TPR threshold (task-priority class only).
3402	*/
3403	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3404	{
3405	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3406	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3407	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3408	}
3409
3410
3411	/**
3412	* Loads the guest APIC and related state.
3413	*
3414	* @returns VBox status code.
3415	* @param pVCpu The cross context virtual CPU structure.
3416	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3417	* out-of-sync. Make sure to update the required fields
3418	* before using them.
3419	*
3420	* @remarks No-long-jump zone!!!
3421	*/
3422	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3423	{
3424	NOREF(pMixedCtx);
3425
3426	int rc = VINF_SUCCESS;
3427	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3428	{
3429	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3430	&& APICIsEnabled(pVCpu))
3431	{
3432	/*
3433	* Setup TPR shadowing.
3434	*/
3435	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3436	{
3437	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3438
3439	bool fPendingIntr = false;
3440	uint8_t u8Tpr = 0;
3441	uint8_t u8PendingIntr = 0;
3442	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3443	AssertRCReturn(rc, rc);
3444
3445	/*
3446	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3447	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3448	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3449	*/
3450	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3451	uint32_t u32TprThreshold = 0;
3452	if (fPendingIntr)
3453	{
3454	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3455	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3456	const uint8_t u8TprPriority = u8Tpr >> 4;
3457	if (u8PendingPriority <= u8TprPriority)
3458	u32TprThreshold = u8PendingPriority;
3459	}
3460
3461	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3462	AssertRCReturn(rc, rc);
3463	}
3464	}
3465	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3466	}
3467
3468	return rc;
3469	}
3470
3471
3472	/**
3473	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3474	*
3475	* @returns Guest's interruptibility-state.
3476	* @param pVCpu The cross context virtual CPU structure.
3477	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3478	* out-of-sync. Make sure to update the required fields
3479	* before using them.
3480	*
3481	* @remarks No-long-jump zone!!!
3482	*/
3483	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3484	{
3485	/*
3486	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3487	*/
3488	uint32_t uIntrState = 0;
3489	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3490	{
3491	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3492	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3493	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3494	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3495	{
3496	if (pMixedCtx->eflags.Bits.u1IF)
3497	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3498	else
3499	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3500	}
3501	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3502	{
3503	/*
3504	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3505	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3506	*/
3507	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3508	}
3509	}
3510
3511	/*
3512	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3513	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3514	* setting this would block host-NMIs and IRET will not clear the blocking.
3515	*
3516	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3517	*/
3518	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3519	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3520	{
3521	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3522	}
3523
3524	return uIntrState;
3525	}
3526
3527
3528	/**
3529	* Loads the guest's interruptibility-state into the guest-state area in the
3530	* VMCS.
3531	*
3532	* @returns VBox status code.
3533	* @param pVCpu The cross context virtual CPU structure.
3534	* @param uIntrState The interruptibility-state to set.
3535	*/
3536	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3537	{
3538	NOREF(pVCpu);
3539	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3540	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3541	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3542	AssertRC(rc);
3543	return rc;
3544	}
3545
3546
3547	/**
3548	* Loads the exception intercepts required for guest execution in the VMCS.
3549	*
3550	* @returns VBox status code.
3551	* @param pVCpu The cross context virtual CPU structure.
3552	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3553	* out-of-sync. Make sure to update the required fields
3554	* before using them.
3555	*/
3556	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3557	{
3558	NOREF(pMixedCtx);
3559	int rc = VINF_SUCCESS;
3560	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3561	{
3562	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3563	if (pVCpu->hm.s.fGIMTrapXcptUD)
3564	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3565	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3566	else
3567	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3568	#endif
3569
3570	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3571	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3572
3573	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3574	AssertRCReturn(rc, rc);
3575
3576	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3577	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3578	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3579	}
3580	return rc;
3581	}
3582
3583
3584	/**
3585	* Loads the guest's RIP into the guest-state area in the VMCS.
3586	*
3587	* @returns VBox status code.
3588	* @param pVCpu The cross context virtual CPU structure.
3589	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590	* out-of-sync. Make sure to update the required fields
3591	* before using them.
3592	*
3593	* @remarks No-long-jump zone!!!
3594	*/
3595	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596	{
3597	int rc = VINF_SUCCESS;
3598	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3599	{
3600	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3601	AssertRCReturn(rc, rc);
3602
3603	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3604	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3605	HMCPU_CF_VALUE(pVCpu)));
3606	}
3607	return rc;
3608	}
3609
3610
3611	/**
3612	* Loads the guest's RSP into the guest-state area in the VMCS.
3613	*
3614	* @returns VBox status code.
3615	* @param pVCpu The cross context virtual CPU structure.
3616	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3617	* out-of-sync. Make sure to update the required fields
3618	* before using them.
3619	*
3620	* @remarks No-long-jump zone!!!
3621	*/
3622	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3623	{
3624	int rc = VINF_SUCCESS;
3625	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3626	{
3627	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3628	AssertRCReturn(rc, rc);
3629
3630	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3631	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3632	}
3633	return rc;
3634	}
3635
3636
3637	/**
3638	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3639	*
3640	* @returns VBox status code.
3641	* @param pVCpu The cross context virtual CPU structure.
3642	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3643	* out-of-sync. Make sure to update the required fields
3644	* before using them.
3645	*
3646	* @remarks No-long-jump zone!!!
3647	*/
3648	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3649	{
3650	int rc = VINF_SUCCESS;
3651	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3652	{
3653	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3654	Let us assert it as such and use 32-bit VMWRITE. */
3655	Assert(!(pMixedCtx->rflags.u64 >> 32));
3656	X86EFLAGS Eflags = pMixedCtx->eflags;
3657	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3658	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3659	* These will never be cleared/set, unless some other part of the VMM
3660	* code is buggy - in which case we're better of finding and fixing
3661	* those bugs than hiding them. */
3662	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3663	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3664	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3665	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3666
3667	/*
3668	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3669	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3670	*/
3671	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3672	{
3673	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3674	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3675	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3676	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3677	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3678	}
3679
3680	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3681	AssertRCReturn(rc, rc);
3682
3683	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3684	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3685	}
3686	return rc;
3687	}
3688
3689
3690	/**
3691	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3692	*
3693	* @returns VBox status code.
3694	* @param pVCpu The cross context virtual CPU structure.
3695	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3696	* out-of-sync. Make sure to update the required fields
3697	* before using them.
3698	*
3699	* @remarks No-long-jump zone!!!
3700	*/
3701	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3702	{
3703	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3704	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3705	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3706	AssertRCReturn(rc, rc);
3707	return rc;
3708	}
3709
3710
3711	/**
3712	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3713	* CR0 is partially shared with the host and we have to consider the FPU bits.
3714	*
3715	* @returns VBox status code.
3716	* @param pVCpu The cross context virtual CPU structure.
3717	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3718	* out-of-sync. Make sure to update the required fields
3719	* before using them.
3720	*
3721	* @remarks No-long-jump zone!!!
3722	*/
3723	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3724	{
3725	/*
3726	* Guest CR0.
3727	* Guest FPU.
3728	*/
3729	int rc = VINF_SUCCESS;
3730	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3731	{
3732	Assert(!(pMixedCtx->cr0 >> 32));
3733	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3734	PVM pVM = pVCpu->CTX_SUFF(pVM);
3735
3736	/* The guest's view (read access) of its CR0 is unblemished. */
3737	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3738	AssertRCReturn(rc, rc);
3739	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3740
3741	/* Setup VT-x's view of the guest CR0. */
3742	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3743	if (pVM->hm.s.fNestedPaging)
3744	{
3745	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3746	{
3747	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3748	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3749	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3750	}
3751	else
3752	{
3753	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3754	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3755	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3756	}
3757
3758	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3759	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3760	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3761
3762	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3763	AssertRCReturn(rc, rc);
3764	}
3765	else
3766	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3767
3768	/*
3769	* Guest FPU bits.
3770	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3771	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3772	*/
3773	u32GuestCR0 \|= X86_CR0_NE;
3774	bool fInterceptNM = false;
3775	if (CPUMIsGuestFPUStateActive(pVCpu))
3776	{
3777	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3778	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3779	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3780	}
3781	else
3782	{
3783	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3784	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3785	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3786	}
3787
3788	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3789	bool fInterceptMF = false;
3790	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3791	fInterceptMF = true;
3792
3793	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3794	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3795	{
3796	Assert(PDMVmmDevHeapIsEnabled(pVM));
3797	Assert(pVM->hm.s.vmx.pRealModeTSS);
3798	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3799	fInterceptNM = true;
3800	fInterceptMF = true;
3801	}
3802	else
3803	{
3804	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3805	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3806	}
3807	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3808
3809	if (fInterceptNM)
3810	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3811	else
3812	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3813
3814	if (fInterceptMF)
3815	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3816	else
3817	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3818
3819	/* Additional intercepts for debugging, define these yourself explicitly. */
3820	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3821	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3822	\| RT_BIT(X86_XCPT_BP)
3823	\| RT_BIT(X86_XCPT_DE)
3824	\| RT_BIT(X86_XCPT_NM)
3825	\| RT_BIT(X86_XCPT_TS)
3826	\| RT_BIT(X86_XCPT_UD)
3827	\| RT_BIT(X86_XCPT_NP)
3828	\| RT_BIT(X86_XCPT_SS)
3829	\| RT_BIT(X86_XCPT_GP)
3830	\| RT_BIT(X86_XCPT_PF)
3831	\| RT_BIT(X86_XCPT_MF)
3832	;
3833	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3834	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3835	#endif
3836
3837	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3838
3839	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3840	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3841	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3842	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3843	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3844	else
3845	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3846
3847	u32GuestCR0 \|= uSetCR0;
3848	u32GuestCR0 &= uZapCR0;
3849	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3850
3851	/* Write VT-x's view of the guest CR0 into the VMCS. */
3852	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3853	AssertRCReturn(rc, rc);
3854	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3855	uZapCR0));
3856
3857	/*
3858	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3859	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3860	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3861	*/
3862	uint32_t u32CR0Mask = 0;
3863	u32CR0Mask = X86_CR0_PE
3864	\| X86_CR0_NE
3865	\| X86_CR0_WP
3866	\| X86_CR0_PG
3867	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3868	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3869	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3870
3871	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3872	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3873	* and @bugref{6944}. */
3874	#if 0
3875	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3876	u32CR0Mask &= ~X86_CR0_PE;
3877	#endif
3878	if (pVM->hm.s.fNestedPaging)
3879	u32CR0Mask &= ~X86_CR0_WP;
3880
3881	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3882	if (fInterceptNM)
3883	{
3884	u32CR0Mask \|= X86_CR0_TS
3885	\| X86_CR0_MP;
3886	}
3887
3888	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3889	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3890	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3891	AssertRCReturn(rc, rc);
3892	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3893
3894	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3895	}
3896	return rc;
3897	}
3898
3899
3900	/**
3901	* Loads the guest control registers (CR3, CR4) into the guest-state area
3902	* in the VMCS.
3903	*
3904	* @returns VBox strict status code.
3905	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3906	* without unrestricted guest access and the VMMDev is not presently
3907	* mapped (e.g. EFI32).
3908	*
3909	* @param pVCpu The cross context virtual CPU structure.
3910	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3911	* out-of-sync. Make sure to update the required fields
3912	* before using them.
3913	*
3914	* @remarks No-long-jump zone!!!
3915	*/
3916	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3917	{
3918	int rc = VINF_SUCCESS;
3919	PVM pVM = pVCpu->CTX_SUFF(pVM);
3920
3921	/*
3922	* Guest CR2.
3923	* It's always loaded in the assembler code. Nothing to do here.
3924	*/
3925
3926	/*
3927	* Guest CR3.
3928	*/
3929	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3930	{
3931	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3932	if (pVM->hm.s.fNestedPaging)
3933	{
3934	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3935
3936	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3937	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3938	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3939	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3940
3941	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3942	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3943	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3944
3945	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3946	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3947	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3948	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3949	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3950	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3951	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3952
3953	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3954	AssertRCReturn(rc, rc);
3955	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3956
3957	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3958	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3959	{
3960	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3961	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3962	{
3963	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3964	AssertRCReturn(rc, rc);
3965	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3966	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3967	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3968	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3969	AssertRCReturn(rc, rc);
3970	}
3971
3972	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3973	have Unrestricted Execution to handle the guest when it's not using paging. */
3974	GCPhysGuestCR3 = pMixedCtx->cr3;
3975	}
3976	else
3977	{
3978	/*
3979	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3980	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3981	* EPT takes care of translating it to host-physical addresses.
3982	*/
3983	RTGCPHYS GCPhys;
3984	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3985
3986	/* We obtain it here every time as the guest could have relocated this PCI region. */
3987	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3988	if (RT_SUCCESS(rc))
3989	{ /* likely */ }
3990	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3991	{
3992	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3993	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3994	}
3995	else
3996	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3997
3998	GCPhysGuestCR3 = GCPhys;
3999	}
4000
4001	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4002	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4003	}
4004	else
4005	{
4006	/* Non-nested paging case, just use the hypervisor's CR3. */
4007	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4008
4009	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4010	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4011	}
4012	AssertRCReturn(rc, rc);
4013
4014	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4015	}
4016
4017	/*
4018	* Guest CR4.
4019	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4020	*/
4021	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4022	{
4023	Assert(!(pMixedCtx->cr4 >> 32));
4024	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4025
4026	/* The guest's view of its CR4 is unblemished. */
4027	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4028	AssertRCReturn(rc, rc);
4029	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4030
4031	/* Setup VT-x's view of the guest CR4. */
4032	/*
4033	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4034	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4035	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4036	*/
4037	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4038	{
4039	Assert(pVM->hm.s.vmx.pRealModeTSS);
4040	Assert(PDMVmmDevHeapIsEnabled(pVM));
4041	u32GuestCR4 &= ~X86_CR4_VME;
4042	}
4043
4044	if (pVM->hm.s.fNestedPaging)
4045	{
4046	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4047	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4048	{
4049	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4050	u32GuestCR4 \|= X86_CR4_PSE;
4051	/* Our identity mapping is a 32-bit page directory. */
4052	u32GuestCR4 &= ~X86_CR4_PAE;
4053	}
4054	/* else use guest CR4.*/
4055	}
4056	else
4057	{
4058	/*
4059	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4060	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4061	*/
4062	switch (pVCpu->hm.s.enmShadowMode)
4063	{
4064	case PGMMODE_REAL: /* Real-mode. */
4065	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4066	case PGMMODE_32_BIT: /* 32-bit paging. */
4067	{
4068	u32GuestCR4 &= ~X86_CR4_PAE;
4069	break;
4070	}
4071
4072	case PGMMODE_PAE: /* PAE paging. */
4073	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4074	{
4075	u32GuestCR4 \|= X86_CR4_PAE;
4076	break;
4077	}
4078
4079	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4080	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4081	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4082	break;
4083	#endif
4084	default:
4085	AssertFailed();
4086	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4087	}
4088	}
4089
4090	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4091	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4092	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4093	u32GuestCR4 \|= uSetCR4;
4094	u32GuestCR4 &= uZapCR4;
4095
4096	/* Write VT-x's view of the guest CR4 into the VMCS. */
4097	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4098	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4099	AssertRCReturn(rc, rc);
4100
4101	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4102	uint32_t u32CR4Mask = X86_CR4_VME
4103	\| X86_CR4_PAE
4104	\| X86_CR4_PGE
4105	\| X86_CR4_PSE
4106	\| X86_CR4_VMXE;
4107	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4108	u32CR4Mask \|= X86_CR4_OSXSAVE;
4109	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4110	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4111	AssertRCReturn(rc, rc);
4112
4113	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4114	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4115
4116	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4117	}
4118	return rc;
4119	}
4120
4121
4122	/**
4123	* Loads the guest debug registers into the guest-state area in the VMCS.
4124	*
4125	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4126	*
4127	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4128	*
4129	* @returns VBox status code.
4130	* @param pVCpu The cross context virtual CPU structure.
4131	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4132	* out-of-sync. Make sure to update the required fields
4133	* before using them.
4134	*
4135	* @remarks No-long-jump zone!!!
4136	*/
4137	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4138	{
4139	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4140	return VINF_SUCCESS;
4141
4142	#ifdef VBOX_STRICT
4143	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4144	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4145	{
4146	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4147	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4148	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4149	}
4150	#endif
4151
4152	int rc;
4153	PVM pVM = pVCpu->CTX_SUFF(pVM);
4154	bool fSteppingDB = false;
4155	bool fInterceptMovDRx = false;
4156	if (pVCpu->hm.s.fSingleInstruction)
4157	{
4158	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4159	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4160	{
4161	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4162	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4163	AssertRCReturn(rc, rc);
4164	Assert(fSteppingDB == false);
4165	}
4166	else
4167	{
4168	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4169	pVCpu->hm.s.fClearTrapFlag = true;
4170	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4171	fSteppingDB = true;
4172	}
4173	}
4174
4175	if ( fSteppingDB
4176	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4177	{
4178	/*
4179	* Use the combined guest and host DRx values found in the hypervisor
4180	* register set because the debugger has breakpoints active or someone
4181	* is single stepping on the host side without a monitor trap flag.
4182	*
4183	* Note! DBGF expects a clean DR6 state before executing guest code.
4184	*/
4185	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4186	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4187	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4188	{
4189	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4190	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4191	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4192	}
4193	else
4194	#endif
4195	if (!CPUMIsHyperDebugStateActive(pVCpu))
4196	{
4197	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4198	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4199	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4200	}
4201
4202	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4203	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4204	AssertRCReturn(rc, rc);
4205
4206	pVCpu->hm.s.fUsingHyperDR7 = true;
4207	fInterceptMovDRx = true;
4208	}
4209	else
4210	{
4211	/*
4212	* If the guest has enabled debug registers, we need to load them prior to
4213	* executing guest code so they'll trigger at the right time.
4214	*/
4215	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4216	{
4217	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4218	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4219	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4220	{
4221	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4222	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4223	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4224	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4225	}
4226	else
4227	#endif
4228	if (!CPUMIsGuestDebugStateActive(pVCpu))
4229	{
4230	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4231	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4232	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4233	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4234	}
4235	Assert(!fInterceptMovDRx);
4236	}
4237	/*
4238	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4239	* must intercept #DB in order to maintain a correct DR6 guest value, and
4240	* because we need to intercept it to prevent nested #DBs from hanging the
4241	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4242	*/
4243	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4244	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4245	&& !CPUMIsGuestDebugStateActive(pVCpu))
4246	#else
4247	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4248	#endif
4249	{
4250	fInterceptMovDRx = true;
4251	}
4252
4253	/* Update guest DR7. */
4254	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4255	AssertRCReturn(rc, rc);
4256
4257	pVCpu->hm.s.fUsingHyperDR7 = false;
4258	}
4259
4260	/*
4261	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4262	*/
4263	if (fInterceptMovDRx)
4264	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4265	else
4266	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4267	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4268	AssertRCReturn(rc, rc);
4269
4270	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4271	return VINF_SUCCESS;
4272	}
4273
4274
4275	#ifdef VBOX_STRICT
4276	/**
4277	* Strict function to validate segment registers.
4278	*
4279	* @remarks ASSUMES CR0 is up to date.
4280	*/
4281	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4282	{
4283	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4284	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4285	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4286	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4287	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4288	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4289	{
4290	/* Protected mode checks */
4291	/* CS */
4292	Assert(pCtx->cs.Attr.n.u1Present);
4293	Assert(!(pCtx->cs.Attr.u & 0xf00));
4294	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4295	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4296	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4297	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4298	\|\| (pCtx->cs.Attr.n.u1Granularity));
4299	/* CS cannot be loaded with NULL in protected mode. */
4300	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4301	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4302	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4303	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4304	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4305	else
4306	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4307	/* SS */
4308	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4309	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4310	if ( !(pCtx->cr0 & X86_CR0_PE)
4311	\|\| pCtx->cs.Attr.n.u4Type == 3)
4312	{
4313	Assert(!pCtx->ss.Attr.n.u2Dpl);
4314	}
4315	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4316	{
4317	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4318	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4319	Assert(pCtx->ss.Attr.n.u1Present);
4320	Assert(!(pCtx->ss.Attr.u & 0xf00));
4321	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4322	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4323	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4324	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4325	\|\| (pCtx->ss.Attr.n.u1Granularity));
4326	}
4327	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4328	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4329	{
4330	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4331	Assert(pCtx->ds.Attr.n.u1Present);
4332	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4333	Assert(!(pCtx->ds.Attr.u & 0xf00));
4334	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4335	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4336	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4337	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4338	\|\| (pCtx->ds.Attr.n.u1Granularity));
4339	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4340	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4341	}
4342	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4343	{
4344	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4345	Assert(pCtx->es.Attr.n.u1Present);
4346	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4347	Assert(!(pCtx->es.Attr.u & 0xf00));
4348	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4349	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4350	\|\| !(pCtx->es.Attr.n.u1Granularity));
4351	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4352	\|\| (pCtx->es.Attr.n.u1Granularity));
4353	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4354	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4355	}
4356	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4357	{
4358	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4359	Assert(pCtx->fs.Attr.n.u1Present);
4360	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4361	Assert(!(pCtx->fs.Attr.u & 0xf00));
4362	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4363	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4364	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4365	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4366	\|\| (pCtx->fs.Attr.n.u1Granularity));
4367	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4368	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4369	}
4370	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4371	{
4372	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4373	Assert(pCtx->gs.Attr.n.u1Present);
4374	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4375	Assert(!(pCtx->gs.Attr.u & 0xf00));
4376	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4377	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4378	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4379	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4380	\|\| (pCtx->gs.Attr.n.u1Granularity));
4381	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4382	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4383	}
4384	/* 64-bit capable CPUs. */
4385	# if HC_ARCH_BITS == 64
4386	Assert(!(pCtx->cs.u64Base >> 32));
4387	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4388	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4389	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4390	# endif
4391	}
4392	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4393	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4394	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4395	{
4396	/* Real and v86 mode checks. */
4397	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4398	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4399	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4400	{
4401	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4402	}
4403	else
4404	{
4405	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4406	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4407	}
4408
4409	/* CS */
4410	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4411	Assert(pCtx->cs.u32Limit == 0xffff);
4412	Assert(u32CSAttr == 0xf3);
4413	/* SS */
4414	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4415	Assert(pCtx->ss.u32Limit == 0xffff);
4416	Assert(u32SSAttr == 0xf3);
4417	/* DS */
4418	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4419	Assert(pCtx->ds.u32Limit == 0xffff);
4420	Assert(u32DSAttr == 0xf3);
4421	/* ES */
4422	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4423	Assert(pCtx->es.u32Limit == 0xffff);
4424	Assert(u32ESAttr == 0xf3);
4425	/* FS */
4426	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4427	Assert(pCtx->fs.u32Limit == 0xffff);
4428	Assert(u32FSAttr == 0xf3);
4429	/* GS */
4430	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4431	Assert(pCtx->gs.u32Limit == 0xffff);
4432	Assert(u32GSAttr == 0xf3);
4433	/* 64-bit capable CPUs. */
4434	# if HC_ARCH_BITS == 64
4435	Assert(!(pCtx->cs.u64Base >> 32));
4436	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4437	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4438	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4439	# endif
4440	}
4441	}
4442	#endif /* VBOX_STRICT */
4443
4444
4445	/**
4446	* Writes a guest segment register into the guest-state area in the VMCS.
4447	*
4448	* @returns VBox status code.
4449	* @param pVCpu The cross context virtual CPU structure.
4450	* @param idxSel Index of the selector in the VMCS.
4451	* @param idxLimit Index of the segment limit in the VMCS.
4452	* @param idxBase Index of the segment base in the VMCS.
4453	* @param idxAccess Index of the access rights of the segment in the VMCS.
4454	* @param pSelReg Pointer to the segment selector.
4455	*
4456	* @remarks No-long-jump zone!!!
4457	*/
4458	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4459	uint32_t idxAccess, PCPUMSELREG pSelReg)
4460	{
4461	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4462	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4463	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4464	AssertRCReturn(rc, rc);
4465
4466	uint32_t u32Access = pSelReg->Attr.u;
4467	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4468	{
4469	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4470	u32Access = 0xf3;
4471	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4472	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4473	}
4474	else
4475	{
4476	/*
4477	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4478	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4479	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4480	* loaded in protected-mode have their attribute as 0.
4481	*/
4482	if (!u32Access)
4483	u32Access = X86DESCATTR_UNUSABLE;
4484	}
4485
4486	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4487	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4488	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4489
4490	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4491	AssertRCReturn(rc, rc);
4492	return rc;
4493	}
4494
4495
4496	/**
4497	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4498	* into the guest-state area in the VMCS.
4499	*
4500	* @returns VBox status code.
4501	* @param pVCpu The cross context virtual CPU structure.
4502	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4503	* out-of-sync. Make sure to update the required fields
4504	* before using them.
4505	*
4506	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4507	* @remarks No-long-jump zone!!!
4508	*/
4509	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4510	{
4511	int rc = VERR_INTERNAL_ERROR_5;
4512	PVM pVM = pVCpu->CTX_SUFF(pVM);
4513
4514	/*
4515	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4516	*/
4517	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4518	{
4519	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4520	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4521	{
4522	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4523	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4524	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4525	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4526	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4527	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4528	}
4529
4530	#ifdef VBOX_WITH_REM
4531	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4532	{
4533	Assert(pVM->hm.s.vmx.pRealModeTSS);
4534	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4535	if ( pVCpu->hm.s.vmx.fWasInRealMode
4536	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4537	{
4538	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4539	in real-mode (e.g. OpenBSD 4.0) */
4540	REMFlushTBs(pVM);
4541	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4542	pVCpu->hm.s.vmx.fWasInRealMode = false;
4543	}
4544	}
4545	#endif
4546	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4547	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4548	AssertRCReturn(rc, rc);
4549	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4550	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4551	AssertRCReturn(rc, rc);
4552	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4553	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4554	AssertRCReturn(rc, rc);
4555	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4556	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4557	AssertRCReturn(rc, rc);
4558	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4559	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4560	AssertRCReturn(rc, rc);
4561	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4562	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4563	AssertRCReturn(rc, rc);
4564
4565	#ifdef VBOX_STRICT
4566	/* Validate. */
4567	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4568	#endif
4569
4570	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4571	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4572	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4573	}
4574
4575	/*
4576	* Guest TR.
4577	*/
4578	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4579	{
4580	/*
4581	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4582	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4583	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4584	*/
4585	uint16_t u16Sel = 0;
4586	uint32_t u32Limit = 0;
4587	uint64_t u64Base = 0;
4588	uint32_t u32AccessRights = 0;
4589
4590	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4591	{
4592	u16Sel = pMixedCtx->tr.Sel;
4593	u32Limit = pMixedCtx->tr.u32Limit;
4594	u64Base = pMixedCtx->tr.u64Base;
4595	u32AccessRights = pMixedCtx->tr.Attr.u;
4596	}
4597	else
4598	{
4599	Assert(pVM->hm.s.vmx.pRealModeTSS);
4600	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4601
4602	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4603	RTGCPHYS GCPhys;
4604	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4605	AssertRCReturn(rc, rc);
4606
4607	X86DESCATTR DescAttr;
4608	DescAttr.u = 0;
4609	DescAttr.n.u1Present = 1;
4610	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4611
4612	u16Sel = 0;
4613	u32Limit = HM_VTX_TSS_SIZE;
4614	u64Base = GCPhys; /* in real-mode phys = virt. */
4615	u32AccessRights = DescAttr.u;
4616	}
4617
4618	/* Validate. */
4619	Assert(!(u16Sel & RT_BIT(2)));
4620	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4621	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4622	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4623	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4624	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4625	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4626	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4627	Assert( (u32Limit & 0xfff) == 0xfff
4628	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4629	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4630	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4631
4632	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4633	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4634	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4635	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4636	AssertRCReturn(rc, rc);
4637
4638	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4639	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4640	}
4641
4642	/*
4643	* Guest GDTR.
4644	*/
4645	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4646	{
4647	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4648	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4649	AssertRCReturn(rc, rc);
4650
4651	/* Validate. */
4652	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4653
4654	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4655	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4656	}
4657
4658	/*
4659	* Guest LDTR.
4660	*/
4661	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4662	{
4663	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4664	uint32_t u32Access = 0;
4665	if (!pMixedCtx->ldtr.Attr.u)
4666	u32Access = X86DESCATTR_UNUSABLE;
4667	else
4668	u32Access = pMixedCtx->ldtr.Attr.u;
4669
4670	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4671	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4672	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4673	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4674	AssertRCReturn(rc, rc);
4675
4676	/* Validate. */
4677	if (!(u32Access & X86DESCATTR_UNUSABLE))
4678	{
4679	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4680	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4681	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4682	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4683	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4684	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4685	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4686	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4687	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4688	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4689	}
4690
4691	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4692	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4693	}
4694
4695	/*
4696	* Guest IDTR.
4697	*/
4698	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4699	{
4700	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4701	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4702	AssertRCReturn(rc, rc);
4703
4704	/* Validate. */
4705	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4706
4707	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4708	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4709	}
4710
4711	return VINF_SUCCESS;
4712	}
4713
4714
4715	/**
4716	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4717	* areas.
4718	*
4719	* These MSRs will automatically be loaded to the host CPU on every successful
4720	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4721	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4722	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4723	*
4724	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4725	*
4726	* @returns VBox status code.
4727	* @param pVCpu The cross context virtual CPU structure.
4728	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4729	* out-of-sync. Make sure to update the required fields
4730	* before using them.
4731	*
4732	* @remarks No-long-jump zone!!!
4733	*/
4734	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4735	{
4736	AssertPtr(pVCpu);
4737	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4738
4739	/*
4740	* MSRs that we use the auto-load/store MSR area in the VMCS.
4741	*/
4742	PVM pVM = pVCpu->CTX_SUFF(pVM);
4743	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4744	{
4745	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4746	#if HC_ARCH_BITS == 32
4747	if (pVM->hm.s.fAllow64BitGuests)
4748	{
4749	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4750	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4751	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4752	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4753	AssertRCReturn(rc, rc);
4754	# ifdef LOG_ENABLED
4755	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4756	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4757	{
4758	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4759	pMsr->u64Value));
4760	}
4761	# endif
4762	}
4763	#endif
4764	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4765	}
4766
4767	/*
4768	* Guest Sysenter MSRs.
4769	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4770	* VM-exits on WRMSRs for these MSRs.
4771	*/
4772	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4773	{
4774	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4775	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4776	}
4777
4778	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4779	{
4780	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4781	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4782	}
4783
4784	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4785	{
4786	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4787	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4788	}
4789
4790	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4791	{
4792	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4793	{
4794	/*
4795	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4796	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4797	*/
4798	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4799	{
4800	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4801	AssertRCReturn(rc,rc);
4802	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4803	}
4804	else
4805	{
4806	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4807	NULL /* pfAddedAndUpdated */);
4808	AssertRCReturn(rc, rc);
4809
4810	/* We need to intercept reads too, see @bugref{7386#c16}. */
4811	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4812	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4813	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4814	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4815	}
4816	}
4817	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4818	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4819	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4820	}
4821
4822	return VINF_SUCCESS;
4823	}
4824
4825
4826	/**
4827	* Loads the guest activity state into the guest-state area in the VMCS.
4828	*
4829	* @returns VBox status code.
4830	* @param pVCpu The cross context virtual CPU structure.
4831	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4832	* out-of-sync. Make sure to update the required fields
4833	* before using them.
4834	*
4835	* @remarks No-long-jump zone!!!
4836	*/
4837	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4838	{
4839	NOREF(pMixedCtx);
4840	/** @todo See if we can make use of other states, e.g.
4841	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4842	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4843	{
4844	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4845	AssertRCReturn(rc, rc);
4846
4847	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4848	}
4849	return VINF_SUCCESS;
4850	}
4851
4852
4853	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4854	/**
4855	* Check if guest state allows safe use of 32-bit switcher again.
4856	*
4857	* Segment bases and protected mode structures must be 32-bit addressable
4858	* because the 32-bit switcher will ignore high dword when writing these VMCS
4859	* fields. See @bugref{8432} for details.
4860	*
4861	* @returns true if safe, false if must continue to use the 64-bit switcher.
4862	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4863	* out-of-sync. Make sure to update the required fields
4864	* before using them.
4865	*
4866	* @remarks No-long-jump zone!!!
4867	*/
4868	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4869	{
4870	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4871	return false;
4872	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4873	return false;
4874	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4875	return false;
4876	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4877	return false;
4878	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4879	return false;
4880	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4881	return false;
4882	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4883	return false;
4884	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4885	return false;
4886	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4887	return false;
4888	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4889	return false;
4890	/* All good, bases are 32-bit. */
4891	return true;
4892	}
4893	#endif
4894
4895
4896	/**
4897	* Sets up the appropriate function to run guest code.
4898	*
4899	* @returns VBox status code.
4900	* @param pVCpu The cross context virtual CPU structure.
4901	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4902	* out-of-sync. Make sure to update the required fields
4903	* before using them.
4904	*
4905	* @remarks No-long-jump zone!!!
4906	*/
4907	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4908	{
4909	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4910	{
4911	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4912	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4913	#endif
4914	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4915	#if HC_ARCH_BITS == 32
4916	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4917	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4918	{
4919	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4920	{
4921	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4922	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4923	\| HM_CHANGED_VMX_ENTRY_CTLS
4924	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4925	}
4926	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4927
4928	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4929	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4930	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4931	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4932	}
4933	#else
4934	/* 64-bit host. */
4935	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4936	#endif
4937	}
4938	else
4939	{
4940	/* Guest is not in long mode, use the 32-bit handler. */
4941	#if HC_ARCH_BITS == 32
4942	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4943	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4944	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4945	{
4946	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4947	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4948	\| HM_CHANGED_VMX_ENTRY_CTLS
4949	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4950	}
4951	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4952	/*
4953	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4954	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4955	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4956	* the much faster 32-bit switcher again.
4957	*/
4958	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4959	{
4960	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4961	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4962	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4963	}
4964	else
4965	{
4966	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4967	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4968	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4969	{
4970	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4971	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4972	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4973	\| HM_CHANGED_VMX_ENTRY_CTLS
4974	\| HM_CHANGED_VMX_EXIT_CTLS
4975	\| HM_CHANGED_HOST_CONTEXT);
4976	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4977	}
4978	}
4979	# else
4980	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4981	# endif
4982	#else
4983	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4984	#endif
4985	}
4986	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4987	return VINF_SUCCESS;
4988	}
4989
4990
4991	/**
4992	* Wrapper for running the guest code in VT-x.
4993	*
4994	* @returns VBox status code, no informational status codes.
4995	* @param pVM The cross context VM structure.
4996	* @param pVCpu The cross context virtual CPU structure.
4997	* @param pCtx Pointer to the guest-CPU context.
4998	*
4999	* @remarks No-long-jump zone!!!
5000	*/
5001	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5002	{
5003	/*
5004	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5005	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5006	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5007	*/
5008	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5009	/** @todo Add stats for resume vs launch. */
5010	#ifdef VBOX_WITH_KERNEL_USING_XMM
5011	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5012	#else
5013	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5014	#endif
5015	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5016	return rc;
5017	}
5018
5019
5020	/**
5021	* Reports world-switch error and dumps some useful debug info.
5022	*
5023	* @param pVM The cross context VM structure.
5024	* @param pVCpu The cross context virtual CPU structure.
5025	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5026	* @param pCtx Pointer to the guest-CPU context.
5027	* @param pVmxTransient Pointer to the VMX transient structure (only
5028	* exitReason updated).
5029	*/
5030	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5031	{
5032	Assert(pVM);
5033	Assert(pVCpu);
5034	Assert(pCtx);
5035	Assert(pVmxTransient);
5036	HMVMX_ASSERT_PREEMPT_SAFE();
5037
5038	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5039	switch (rcVMRun)
5040	{
5041	case VERR_VMX_INVALID_VMXON_PTR:
5042	AssertFailed();
5043	break;
5044	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5045	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5046	{
5047	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5048	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5049	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5050	AssertRC(rc);
5051
5052	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5053	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5054	Cannot do it here as we may have been long preempted. */
5055
5056	#ifdef VBOX_STRICT
5057	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5058	pVmxTransient->uExitReason));
5059	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5060	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5061	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5062	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5063	else
5064	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5065	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5066	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5067
5068	/* VMX control bits. */
5069	uint32_t u32Val;
5070	uint64_t u64Val;
5071	RTHCUINTREG uHCReg;
5072	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5073	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5074	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5075	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5076	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5077	{
5078	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5079	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5080	}
5081	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5082	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5083	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5084	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5085	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5086	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5088	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5090	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5091	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5092	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5094	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5096	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5097	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5098	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5100	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5101	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5102	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5104	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5106	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5107	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5108	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5109	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5110	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5111	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5112	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5113	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5114	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5115	if (pVM->hm.s.fNestedPaging)
5116	{
5117	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5118	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5119	}
5120
5121	/* Guest bits. */
5122	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5123	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5124	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5125	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5126	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5127	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5128	if (pVM->hm.s.vmx.fVpid)
5129	{
5130	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5131	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5132	}
5133
5134	/* Host bits. */
5135	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5136	Log4(("Host CR0 %#RHr\n", uHCReg));
5137	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5138	Log4(("Host CR3 %#RHr\n", uHCReg));
5139	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5140	Log4(("Host CR4 %#RHr\n", uHCReg));
5141
5142	RTGDTR HostGdtr;
5143	PCX86DESCHC pDesc;
5144	ASMGetGDTR(&HostGdtr);
5145	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5146	Log4(("Host CS %#08x\n", u32Val));
5147	if (u32Val < HostGdtr.cbGdt)
5148	{
5149	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5150	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5151	}
5152
5153	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5154	Log4(("Host DS %#08x\n", u32Val));
5155	if (u32Val < HostGdtr.cbGdt)
5156	{
5157	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5158	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5159	}
5160
5161	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5162	Log4(("Host ES %#08x\n", u32Val));
5163	if (u32Val < HostGdtr.cbGdt)
5164	{
5165	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5166	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5167	}
5168
5169	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5170	Log4(("Host FS %#08x\n", u32Val));
5171	if (u32Val < HostGdtr.cbGdt)
5172	{
5173	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5174	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5175	}
5176
5177	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5178	Log4(("Host GS %#08x\n", u32Val));
5179	if (u32Val < HostGdtr.cbGdt)
5180	{
5181	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5182	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5183	}
5184
5185	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5186	Log4(("Host SS %#08x\n", u32Val));
5187	if (u32Val < HostGdtr.cbGdt)
5188	{
5189	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5190	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5191	}
5192
5193	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5194	Log4(("Host TR %#08x\n", u32Val));
5195	if (u32Val < HostGdtr.cbGdt)
5196	{
5197	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5198	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5199	}
5200
5201	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5202	Log4(("Host TR Base %#RHv\n", uHCReg));
5203	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5204	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5205	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5206	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5207	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5208	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5209	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5210	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5211	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5212	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5213	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5214	Log4(("Host RSP %#RHv\n", uHCReg));
5215	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5216	Log4(("Host RIP %#RHv\n", uHCReg));
5217	# if HC_ARCH_BITS == 64
5218	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5219	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5220	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5221	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5222	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5223	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5224	# endif
5225	#endif /* VBOX_STRICT */
5226	break;
5227	}
5228
5229	default:
5230	/* Impossible */
5231	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5232	break;
5233	}
5234	NOREF(pVM); NOREF(pCtx);
5235	}
5236
5237
5238	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5239	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5240	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5241	#endif
5242	#ifdef VBOX_STRICT
5243	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5244	{
5245	switch (idxField)
5246	{
5247	case VMX_VMCS_GUEST_RIP:
5248	case VMX_VMCS_GUEST_RSP:
5249	case VMX_VMCS_GUEST_SYSENTER_EIP:
5250	case VMX_VMCS_GUEST_SYSENTER_ESP:
5251	case VMX_VMCS_GUEST_GDTR_BASE:
5252	case VMX_VMCS_GUEST_IDTR_BASE:
5253	case VMX_VMCS_GUEST_CS_BASE:
5254	case VMX_VMCS_GUEST_DS_BASE:
5255	case VMX_VMCS_GUEST_ES_BASE:
5256	case VMX_VMCS_GUEST_FS_BASE:
5257	case VMX_VMCS_GUEST_GS_BASE:
5258	case VMX_VMCS_GUEST_SS_BASE:
5259	case VMX_VMCS_GUEST_LDTR_BASE:
5260	case VMX_VMCS_GUEST_TR_BASE:
5261	case VMX_VMCS_GUEST_CR3:
5262	return true;
5263	}
5264	return false;
5265	}
5266
5267	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5268	{
5269	switch (idxField)
5270	{
5271	/* Read-only fields. */
5272	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5273	return true;
5274	}
5275	/* Remaining readable fields should also be writable. */
5276	return hmR0VmxIsValidWriteField(idxField);
5277	}
5278	#endif /* VBOX_STRICT */
5279
5280
5281	/**
5282	* Executes the specified handler in 64-bit mode.
5283	*
5284	* @returns VBox status code (no informational status codes).
5285	* @param pVM The cross context VM structure.
5286	* @param pVCpu The cross context virtual CPU structure.
5287	* @param pCtx Pointer to the guest CPU context.
5288	* @param enmOp The operation to perform.
5289	* @param cParams Number of parameters.
5290	* @param paParam Array of 32-bit parameters.
5291	*/
5292	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5293	uint32_t cParams, uint32_t *paParam)
5294	{
5295	NOREF(pCtx);
5296
5297	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5298	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5299	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5300	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5301
5302	#ifdef VBOX_STRICT
5303	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5304	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5305
5306	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5307	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5308	#endif
5309
5310	/* Disable interrupts. */
5311	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5312
5313	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5314	RTCPUID idHostCpu = RTMpCpuId();
5315	CPUMR0SetLApic(pVCpu, idHostCpu);
5316	#endif
5317
5318	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5319	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5320
5321	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5322	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5323	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5324
5325	/* Leave VMX Root Mode. */
5326	VMXDisable();
5327
5328	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5329
5330	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5331	CPUMSetHyperEIP(pVCpu, enmOp);
5332	for (int i = (int)cParams - 1; i >= 0; i--)
5333	CPUMPushHyper(pVCpu, paParam[i]);
5334
5335	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5336
5337	/* Call the switcher. */
5338	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5339	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5340
5341	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5342	/* Make sure the VMX instructions don't cause #UD faults. */
5343	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5344
5345	/* Re-enter VMX Root Mode */
5346	int rc2 = VMXEnable(HCPhysCpuPage);
5347	if (RT_FAILURE(rc2))
5348	{
5349	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5350	ASMSetFlags(fOldEFlags);
5351	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5352	return rc2;
5353	}
5354
5355	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5356	AssertRC(rc2);
5357	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5358	Assert(!(ASMGetFlags() & X86_EFL_IF));
5359	ASMSetFlags(fOldEFlags);
5360	return rc;
5361	}
5362
5363
5364	/**
5365	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5366	* supporting 64-bit guests.
5367	*
5368	* @returns VBox status code.
5369	* @param fResume Whether to VMLAUNCH or VMRESUME.
5370	* @param pCtx Pointer to the guest-CPU context.
5371	* @param pCache Pointer to the VMCS cache.
5372	* @param pVM The cross context VM structure.
5373	* @param pVCpu The cross context virtual CPU structure.
5374	*/
5375	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5376	{
5377	NOREF(fResume);
5378
5379	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5380	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5381
5382	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5383	pCache->uPos = 1;
5384	pCache->interPD = PGMGetInterPaeCR3(pVM);
5385	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5386	#endif
5387
5388	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5389	pCache->TestIn.HCPhysCpuPage = 0;
5390	pCache->TestIn.HCPhysVmcs = 0;
5391	pCache->TestIn.pCache = 0;
5392	pCache->TestOut.HCPhysVmcs = 0;
5393	pCache->TestOut.pCache = 0;
5394	pCache->TestOut.pCtx = 0;
5395	pCache->TestOut.eflags = 0;
5396	#else
5397	NOREF(pCache);
5398	#endif
5399
5400	uint32_t aParam[10];
5401	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5402	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5403	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5404	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5405	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5406	aParam[5] = 0;
5407	aParam[6] = VM_RC_ADDR(pVM, pVM);
5408	aParam[7] = 0;
5409	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5410	aParam[9] = 0;
5411
5412	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5413	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5414	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5415	#endif
5416	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5417
5418	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5419	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5420	Assert(pCtx->dr[4] == 10);
5421	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5422	#endif
5423
5424	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5425	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5426	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5427	pVCpu->hm.s.vmx.HCPhysVmcs));
5428	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5429	pCache->TestOut.HCPhysVmcs));
5430	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5431	pCache->TestOut.pCache));
5432	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5433	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5434	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5435	pCache->TestOut.pCtx));
5436	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5437	#endif
5438	return rc;
5439	}
5440
5441
5442	/**
5443	* Initialize the VMCS-Read cache.
5444	*
5445	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5446	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5447	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5448	* (those that have a 32-bit FULL & HIGH part).
5449	*
5450	* @returns VBox status code.
5451	* @param pVM The cross context VM structure.
5452	* @param pVCpu The cross context virtual CPU structure.
5453	*/
5454	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5455	{
5456	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5457	{ \
5458	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5459	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5460	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5461	++cReadFields; \
5462	}
5463
5464	AssertPtr(pVM);
5465	AssertPtr(pVCpu);
5466	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5467	uint32_t cReadFields = 0;
5468
5469	/*
5470	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5471	* and serve to indicate exceptions to the rules.
5472	*/
5473
5474	/* Guest-natural selector base fields. */
5475	#if 0
5476	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5477	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5478	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5479	#endif
5480	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5481	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5482	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5483	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5484	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5485	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5486	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5487	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5488	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5489	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5491	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5492	#if 0
5493	/* Unused natural width guest-state fields. */
5494	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5495	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5496	#endif
5497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5498	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5499
5500	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5501	#if 0
5502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5511	#endif
5512
5513	/* Natural width guest-state fields. */
5514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5515	#if 0
5516	/* Currently unused field. */
5517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5518	#endif
5519
5520	if (pVM->hm.s.fNestedPaging)
5521	{
5522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5523	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5524	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5525	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5526	}
5527	else
5528	{
5529	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5530	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5531	}
5532
5533	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5534	return VINF_SUCCESS;
5535	}
5536
5537
5538	/**
5539	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5540	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5541	* darwin, running 64-bit guests).
5542	*
5543	* @returns VBox status code.
5544	* @param pVCpu The cross context virtual CPU structure.
5545	* @param idxField The VMCS field encoding.
5546	* @param u64Val 16, 32 or 64-bit value.
5547	*/
5548	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5549	{
5550	int rc;
5551	switch (idxField)
5552	{
5553	/*
5554	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5555	*/
5556	/* 64-bit Control fields. */
5557	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5558	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5559	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5560	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5561	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5562	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5563	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5564	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5565	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5566	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5567	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5568	case VMX_VMCS64_CTRL_EPTP_FULL:
5569	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5570	/* 64-bit Guest-state fields. */
5571	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5572	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5573	case VMX_VMCS64_GUEST_PAT_FULL:
5574	case VMX_VMCS64_GUEST_EFER_FULL:
5575	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5576	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5577	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5578	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5579	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5580	/* 64-bit Host-state fields. */
5581	case VMX_VMCS64_HOST_PAT_FULL:
5582	case VMX_VMCS64_HOST_EFER_FULL:
5583	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5584	{
5585	rc = VMXWriteVmcs32(idxField, u64Val);
5586	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5587	break;
5588	}
5589
5590	/*
5591	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5592	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5593	*/
5594	/* Natural-width Guest-state fields. */
5595	case VMX_VMCS_GUEST_CR3:
5596	case VMX_VMCS_GUEST_ES_BASE:
5597	case VMX_VMCS_GUEST_CS_BASE:
5598	case VMX_VMCS_GUEST_SS_BASE:
5599	case VMX_VMCS_GUEST_DS_BASE:
5600	case VMX_VMCS_GUEST_FS_BASE:
5601	case VMX_VMCS_GUEST_GS_BASE:
5602	case VMX_VMCS_GUEST_LDTR_BASE:
5603	case VMX_VMCS_GUEST_TR_BASE:
5604	case VMX_VMCS_GUEST_GDTR_BASE:
5605	case VMX_VMCS_GUEST_IDTR_BASE:
5606	case VMX_VMCS_GUEST_RSP:
5607	case VMX_VMCS_GUEST_RIP:
5608	case VMX_VMCS_GUEST_SYSENTER_ESP:
5609	case VMX_VMCS_GUEST_SYSENTER_EIP:
5610	{
5611	if (!(u64Val >> 32))
5612	{
5613	/* If this field is 64-bit, VT-x will zero out the top bits. */
5614	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5615	}
5616	else
5617	{
5618	/* Assert that only the 32->64 switcher case should ever come here. */
5619	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5620	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5621	}
5622	break;
5623	}
5624
5625	default:
5626	{
5627	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5628	rc = VERR_INVALID_PARAMETER;
5629	break;
5630	}
5631	}
5632	AssertRCReturn(rc, rc);
5633	return rc;
5634	}
5635
5636
5637	/**
5638	* Queue up a VMWRITE by using the VMCS write cache.
5639	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5640	*
5641	* @param pVCpu The cross context virtual CPU structure.
5642	* @param idxField The VMCS field encoding.
5643	* @param u64Val 16, 32 or 64-bit value.
5644	*/
5645	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5646	{
5647	AssertPtr(pVCpu);
5648	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5649
5650	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5651	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5652
5653	/* Make sure there are no duplicates. */
5654	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5655	{
5656	if (pCache->Write.aField[i] == idxField)
5657	{
5658	pCache->Write.aFieldVal[i] = u64Val;
5659	return VINF_SUCCESS;
5660	}
5661	}
5662
5663	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5664	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5665	pCache->Write.cValidEntries++;
5666	return VINF_SUCCESS;
5667	}
5668	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5669
5670
5671	/**
5672	* Sets up the usage of TSC-offsetting and updates the VMCS.
5673	*
5674	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5675	* VMX preemption timer.
5676	*
5677	* @returns VBox status code.
5678	* @param pVM The cross context VM structure.
5679	* @param pVCpu The cross context virtual CPU structure.
5680	*
5681	* @remarks No-long-jump zone!!!
5682	*/
5683	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5684	{
5685	int rc;
5686	bool fOffsettedTsc;
5687	bool fParavirtTsc;
5688	if (pVM->hm.s.vmx.fUsePreemptTimer)
5689	{
5690	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5691	&fOffsettedTsc, &fParavirtTsc);
5692
5693	/* Make sure the returned values have sane upper and lower boundaries. */
5694	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5695	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5696	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5697	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5698
5699	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5700	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5701	}
5702	else
5703	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5704
5705	/** @todo later optimize this to be done elsewhere and not before every
5706	* VM-entry. */
5707	if (fParavirtTsc)
5708	{
5709	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5710	information before every VM-entry, hence disable it for performance sake. */
5711	#if 0
5712	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5713	AssertRC(rc);
5714	#endif
5715	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5716	}
5717
5718	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5719	{
5720	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5721	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5722
5723	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5724	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5725	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5726	}
5727	else
5728	{
5729	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5730	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5731	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5732	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5733	}
5734	}
5735
5736
5737	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5738	/**
5739	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5740	* VM-exit interruption info type.
5741	*
5742	* @returns The IEM exception flags.
5743	* @param uVector The event vector.
5744	* @param uVmxVectorType The VMX event type.
5745	*/
5746	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5747	{
5748	uint32_t fIemXcptFlags;
5749	switch (uVmxVectorType)
5750	{
5751	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5752	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5753	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5754	break;
5755
5756	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5757	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5758	break;
5759
5760	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5761	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5762	break;
5763
5764	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5765	{
5766	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5767	if (uVector == X86_XCPT_BP)
5768	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5769	else if (uVector == X86_XCPT_OF)
5770	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5771	else
5772	{
5773	fIemXcptFlags = 0;
5774	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5775	}
5776	break;
5777	}
5778
5779	default:
5780	fIemXcptFlags = 0;
5781	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5782	break;
5783	}
5784	return fIemXcptFlags;
5785	}
5786
5787	#else
5788	/**
5789	* Determines if an exception is a contributory exception.
5790	*
5791	* Contributory exceptions are ones which can cause double-faults unless the
5792	* original exception was a benign exception. Page-fault is intentionally not
5793	* included here as it's a conditional contributory exception.
5794	*
5795	* @returns true if the exception is contributory, false otherwise.
5796	* @param uVector The exception vector.
5797	*/
5798	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5799	{
5800	switch (uVector)
5801	{
5802	case X86_XCPT_GP:
5803	case X86_XCPT_SS:
5804	case X86_XCPT_NP:
5805	case X86_XCPT_TS:
5806	case X86_XCPT_DE:
5807	return true;
5808	default:
5809	break;
5810	}
5811	return false;
5812	}
5813	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5814
5815
5816	/**
5817	* Sets an event as a pending event to be injected into the guest.
5818	*
5819	* @param pVCpu The cross context virtual CPU structure.
5820	* @param u32IntInfo The VM-entry interruption-information field.
5821	* @param cbInstr The VM-entry instruction length in bytes (for software
5822	* interrupts, exceptions and privileged software
5823	* exceptions).
5824	* @param u32ErrCode The VM-entry exception error code.
5825	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5826	* page-fault.
5827	*
5828	* @remarks Statistics counter assumes this is a guest event being injected or
5829	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5830	* always incremented.
5831	*/
5832	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5833	RTGCUINTPTR GCPtrFaultAddress)
5834	{
5835	Assert(!pVCpu->hm.s.Event.fPending);
5836	pVCpu->hm.s.Event.fPending = true;
5837	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5838	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5839	pVCpu->hm.s.Event.cbInstr = cbInstr;
5840	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5841	}
5842
5843
5844	/**
5845	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5846	*
5847	* @param pVCpu The cross context virtual CPU structure.
5848	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5849	* out-of-sync. Make sure to update the required fields
5850	* before using them.
5851	*/
5852	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5853	{
5854	NOREF(pMixedCtx);
5855	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5856	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5857	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5858	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5859	}
5860
5861
5862	/**
5863	* Handle a condition that occurred while delivering an event through the guest
5864	* IDT.
5865	*
5866	* @returns Strict VBox status code (i.e. informational status codes too).
5867	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5868	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5869	* to continue execution of the guest which will delivery the \#DF.
5870	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5871	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5872	*
5873	* @param pVCpu The cross context virtual CPU structure.
5874	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5875	* out-of-sync. Make sure to update the required fields
5876	* before using them.
5877	* @param pVmxTransient Pointer to the VMX transient structure.
5878	*
5879	* @remarks No-long-jump zone!!!
5880	*/
5881	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5882	{
5883	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5884
5885	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5886	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5887
5888	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5889	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5890	{
5891	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5892	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5893	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5894	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5895	IEMXCPTRAISE enmRaise;
5896	IEMXCPTRAISEINFO fRaiseInfo;
5897	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5898	{
5899	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5900	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5901	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5902	AssertMsg( uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
5903	\|\| uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI,
5904	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. type %#x!\n", uExitVectorType));
5905	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector,
5906	&fRaiseInfo);
5907	}
5908	else
5909	{
5910	/*
5911	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5912	* interruption-information will not be valid as it's not an exception and we end up here.
5913	*
5914	* If the event was an external interrupt or hardare exception (incl. NMI) it is sufficient to
5915	* reflect this event to the guest after handling the VM-exit.
5916	*
5917	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5918	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5919	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5920	* event, as that can cause premature trips to ring-3 before injection and involve TRPM which
5921	* currently has no way of storing that the exceptions were caused by these special instructions.
5922	*/
5923	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5924	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5925	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT)
5926	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5927	else
5928	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5929	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5930	}
5931
5932	/*
5933	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5934	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5935	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5936	* subsequent VM-entry would fail.
5937	*
5938	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5939	*/
5940	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5941	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5942	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5943	\|\| fRaiseInfo == IEMXCPTRAISEINFO_NMI_PF)
5944	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5945	{
5946	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5947	}
5948
5949	switch (enmRaise)
5950	{
5951	case IEMXCPTRAISE_CURRENT_XCPT:
5952	{
5953	/*
5954	* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF().
5955	*/
5956	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5957	pVmxTransient->fVectoringPF = true;
5958
5959	/*
5960	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5961	* second #PF as a guest #PF (and not a nested #PF) and needs to be converted into a #DF.
5962	*/
5963	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5964	pVmxTransient->fVectoringDoublePF = true;
5965
5966	Assert(rcStrict == VINF_SUCCESS);
5967	break;
5968	}
5969
5970	case IEMXCPTRAISE_PREV_EVENT:
5971	{
5972	/*
5973	* Re-raise the previous (first) exception/interrupt as delivery caused a premature VM-exit.
5974	*/
5975	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5976	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5977	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5978
5979	uint32_t u32ErrCode;
5980	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5981	{
5982	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5983	AssertRCReturn(rc2, rc2);
5984	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5985	}
5986	else
5987	u32ErrCode = 0;
5988
5989	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5990	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5991	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5992	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5993
5994	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
5995	pVCpu->hm.s.Event.u32ErrCode));
5996	Assert(rcStrict == VINF_SUCCESS);
5997	break;
5998	}
5999
6000	case IEMXCPTRAISE_DOUBLE_FAULT:
6001	{
6002	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6003	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6004
6005	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6006	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6007	rcStrict = VINF_HM_DOUBLE_FAULT;
6008	break;
6009	}
6010
6011	case IEMXCPTRAISE_TRIPLE_FAULT:
6012	{
6013	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6014	uExitVector));
6015	rcStrict = VINF_EM_RESET;
6016	break;
6017	}
6018
6019	case IEMXCPTRAISE_CPU_HANG:
6020	{
6021	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6022	rcStrict = VERR_EM_GUEST_CPU_HANG;
6023	break;
6024	}
6025
6026	case IEMXCPTRAISE_REEXEC_INSTR:
6027	Assert(rcStrict == VINF_SUCCESS);
6028	break;
6029
6030	default:
6031	{
6032	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6033	rcStrict = VERR_VMX_IPE_2;
6034	break;
6035	}
6036	}
6037	#else
6038	typedef enum
6039	{
6040	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6041	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6042	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6043	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6044	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6045	} VMXREFLECTXCPT;
6046
6047	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6048	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6049	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6050	{
6051	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6052	{
6053	enmReflect = VMXREFLECTXCPT_XCPT;
6054	#ifdef VBOX_STRICT
6055	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6056	&& uExitVector == X86_XCPT_PF)
6057	{
6058	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6059	}
6060	#endif
6061	if ( uExitVector == X86_XCPT_PF
6062	&& uIdtVector == X86_XCPT_PF)
6063	{
6064	pVmxTransient->fVectoringDoublePF = true;
6065	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6066	}
6067	else if ( uExitVector == X86_XCPT_AC
6068	&& uIdtVector == X86_XCPT_AC)
6069	{
6070	enmReflect = VMXREFLECTXCPT_HANG;
6071	Log4(("IDT: Nested #AC - Bad guest\n"));
6072	}
6073	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6074	&& hmR0VmxIsContributoryXcpt(uExitVector)
6075	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6076	\|\| uIdtVector == X86_XCPT_PF))
6077	{
6078	enmReflect = VMXREFLECTXCPT_DF;
6079	}
6080	else if (uIdtVector == X86_XCPT_DF)
6081	enmReflect = VMXREFLECTXCPT_TF;
6082	}
6083	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6084	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6085	{
6086	/*
6087	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6088	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6089	*/
6090	enmReflect = VMXREFLECTXCPT_XCPT;
6091
6092	if (uExitVector == X86_XCPT_PF)
6093	{
6094	pVmxTransient->fVectoringPF = true;
6095	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6096	}
6097	}
6098	}
6099	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6100	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6101	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6102	{
6103	/*
6104	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6105	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6106	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6107	*/
6108	enmReflect = VMXREFLECTXCPT_XCPT;
6109	}
6110
6111	/*
6112	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6113	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6114	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6115	*
6116	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6117	*/
6118	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6119	&& enmReflect == VMXREFLECTXCPT_XCPT
6120	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6121	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6122	{
6123	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6124	}
6125
6126	switch (enmReflect)
6127	{
6128	case VMXREFLECTXCPT_XCPT:
6129	{
6130	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6131	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6132	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6133
6134	uint32_t u32ErrCode = 0;
6135	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6136	{
6137	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6138	AssertRCReturn(rc2, rc2);
6139	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6140	}
6141
6142	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6143	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6144	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6145	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6146	rcStrict = VINF_SUCCESS;
6147	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6148	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6149
6150	break;
6151	}
6152
6153	case VMXREFLECTXCPT_DF:
6154	{
6155	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6156	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6157	rcStrict = VINF_HM_DOUBLE_FAULT;
6158	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6159	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6160
6161	break;
6162	}
6163
6164	case VMXREFLECTXCPT_TF:
6165	{
6166	rcStrict = VINF_EM_RESET;
6167	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6168	uExitVector));
6169	break;
6170	}
6171
6172	case VMXREFLECTXCPT_HANG:
6173	{
6174	rcStrict = VERR_EM_GUEST_CPU_HANG;
6175	break;
6176	}
6177
6178	default:
6179	Assert(rcStrict == VINF_SUCCESS);
6180	break;
6181	}
6182	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6183	}
6184	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6185	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6186	&& uExitVector != X86_XCPT_DF
6187	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6188	{
6189	/*
6190	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6191	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6192	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6193	*/
6194	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6195	{
6196	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6197	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6198	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6199	}
6200	}
6201
6202	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6203	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6204	return rcStrict;
6205	}
6206
6207
6208	/**
6209	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6210	*
6211	* @returns VBox status code.
6212	* @param pVCpu The cross context virtual CPU structure.
6213	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6214	* out-of-sync. Make sure to update the required fields
6215	* before using them.
6216	*
6217	* @remarks No-long-jump zone!!!
6218	*/
6219	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6220	{
6221	NOREF(pMixedCtx);
6222
6223	/*
6224	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6225	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6226	*/
6227	VMMRZCallRing3Disable(pVCpu);
6228	HM_DISABLE_PREEMPT();
6229
6230	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6231	{
6232	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6233	* and 'dbgc-init' containing:
6234	* sxe "xcpt_de"
6235	* sxe "xcpt_bp"
6236	* sxi "xcpt_gp"
6237	* sxi "xcpt_ss"
6238	* sxi "xcpt_np"
6239	*/
6240	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6241	#endif
6242	uint32_t uVal = 0;
6243	uint32_t uShadow = 0;
6244	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6245	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6246	AssertRCReturn(rc, rc);
6247
6248	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6249	CPUMSetGuestCR0(pVCpu, uVal);
6250	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6251	}
6252
6253	HM_RESTORE_PREEMPT();
6254	VMMRZCallRing3Enable(pVCpu);
6255	return VINF_SUCCESS;
6256	}
6257
6258
6259	/**
6260	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6261	*
6262	* @returns VBox status code.
6263	* @param pVCpu The cross context virtual CPU structure.
6264	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6265	* out-of-sync. Make sure to update the required fields
6266	* before using them.
6267	*
6268	* @remarks No-long-jump zone!!!
6269	*/
6270	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6271	{
6272	NOREF(pMixedCtx);
6273
6274	int rc = VINF_SUCCESS;
6275	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6276	{
6277	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6278	uint32_t uVal = 0;
6279	uint32_t uShadow = 0;
6280	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6281	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6282	AssertRCReturn(rc, rc);
6283
6284	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6285	CPUMSetGuestCR4(pVCpu, uVal);
6286	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6287	}
6288	return rc;
6289	}
6290
6291
6292	/**
6293	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6294	*
6295	* @returns VBox status code.
6296	* @param pVCpu The cross context virtual CPU structure.
6297	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6298	* out-of-sync. Make sure to update the required fields
6299	* before using them.
6300	*
6301	* @remarks No-long-jump zone!!!
6302	*/
6303	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6304	{
6305	int rc = VINF_SUCCESS;
6306	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6307	{
6308	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6309	uint64_t u64Val = 0;
6310	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6311	AssertRCReturn(rc, rc);
6312
6313	pMixedCtx->rip = u64Val;
6314	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6315	}
6316	return rc;
6317	}
6318
6319
6320	/**
6321	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6322	*
6323	* @returns VBox status code.
6324	* @param pVCpu The cross context virtual CPU structure.
6325	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6326	* out-of-sync. Make sure to update the required fields
6327	* before using them.
6328	*
6329	* @remarks No-long-jump zone!!!
6330	*/
6331	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6332	{
6333	int rc = VINF_SUCCESS;
6334	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6335	{
6336	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6337	uint64_t u64Val = 0;
6338	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6339	AssertRCReturn(rc, rc);
6340
6341	pMixedCtx->rsp = u64Val;
6342	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6343	}
6344	return rc;
6345	}
6346
6347
6348	/**
6349	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6350	*
6351	* @returns VBox status code.
6352	* @param pVCpu The cross context virtual CPU structure.
6353	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6354	* out-of-sync. Make sure to update the required fields
6355	* before using them.
6356	*
6357	* @remarks No-long-jump zone!!!
6358	*/
6359	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6360	{
6361	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6362	{
6363	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6364	uint32_t uVal = 0;
6365	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6366	AssertRCReturn(rc, rc);
6367
6368	pMixedCtx->eflags.u32 = uVal;
6369	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6370	{
6371	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6372	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6373
6374	pMixedCtx->eflags.Bits.u1VM = 0;
6375	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6376	}
6377
6378	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6379	}
6380	return VINF_SUCCESS;
6381	}
6382
6383
6384	/**
6385	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6386	* guest-CPU context.
6387	*/
6388	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6389	{
6390	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6391	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6392	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6393	return rc;
6394	}
6395
6396
6397	/**
6398	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6399	* from the guest-state area in the VMCS.
6400	*
6401	* @param pVCpu The cross context virtual CPU structure.
6402	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6403	* out-of-sync. Make sure to update the required fields
6404	* before using them.
6405	*
6406	* @remarks No-long-jump zone!!!
6407	*/
6408	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6409	{
6410	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6411	{
6412	uint32_t uIntrState = 0;
6413	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6414	AssertRC(rc);
6415
6416	if (!uIntrState)
6417	{
6418	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6419	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6420
6421	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6422	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6423	}
6424	else
6425	{
6426	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6427	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6428	{
6429	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6430	AssertRC(rc);
6431	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6432	AssertRC(rc);
6433
6434	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6435	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6436	}
6437	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6438	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6439
6440	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6441	{
6442	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6443	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6444	}
6445	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6446	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6447	}
6448
6449	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6450	}
6451	}
6452
6453
6454	/**
6455	* Saves the guest's activity state.
6456	*
6457	* @returns VBox status code.
6458	* @param pVCpu The cross context virtual CPU structure.
6459	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6460	* out-of-sync. Make sure to update the required fields
6461	* before using them.
6462	*
6463	* @remarks No-long-jump zone!!!
6464	*/
6465	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6466	{
6467	NOREF(pMixedCtx);
6468	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6469	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6470	return VINF_SUCCESS;
6471	}
6472
6473
6474	/**
6475	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6476	* the current VMCS into the guest-CPU context.
6477	*
6478	* @returns VBox status code.
6479	* @param pVCpu The cross context virtual CPU structure.
6480	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6481	* out-of-sync. Make sure to update the required fields
6482	* before using them.
6483	*
6484	* @remarks No-long-jump zone!!!
6485	*/
6486	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6487	{
6488	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6489	{
6490	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6491	uint32_t u32Val = 0;
6492	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6493	pMixedCtx->SysEnter.cs = u32Val;
6494	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6495	}
6496
6497	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6498	{
6499	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6500	uint64_t u64Val = 0;
6501	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6502	pMixedCtx->SysEnter.eip = u64Val;
6503	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6504	}
6505	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6506	{
6507	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6508	uint64_t u64Val = 0;
6509	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6510	pMixedCtx->SysEnter.esp = u64Val;
6511	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6512	}
6513	return VINF_SUCCESS;
6514	}
6515
6516
6517	/**
6518	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6519	* the CPU back into the guest-CPU context.
6520	*
6521	* @returns VBox status code.
6522	* @param pVCpu The cross context virtual CPU structure.
6523	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6524	* out-of-sync. Make sure to update the required fields
6525	* before using them.
6526	*
6527	* @remarks No-long-jump zone!!!
6528	*/
6529	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6530	{
6531	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6532	VMMRZCallRing3Disable(pVCpu);
6533	HM_DISABLE_PREEMPT();
6534
6535	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6536	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6537	{
6538	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6539	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6540	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6541	}
6542
6543	HM_RESTORE_PREEMPT();
6544	VMMRZCallRing3Enable(pVCpu);
6545
6546	return VINF_SUCCESS;
6547	}
6548
6549
6550	/**
6551	* Saves the auto load/store'd guest MSRs from the current VMCS into
6552	* the guest-CPU context.
6553	*
6554	* @returns VBox status code.
6555	* @param pVCpu The cross context virtual CPU structure.
6556	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6557	* out-of-sync. Make sure to update the required fields
6558	* before using them.
6559	*
6560	* @remarks No-long-jump zone!!!
6561	*/
6562	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6563	{
6564	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6565	return VINF_SUCCESS;
6566
6567	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6568	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6569	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6570	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6571	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6572	{
6573	switch (pMsr->u32Msr)
6574	{
6575	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6576	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6577	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6578	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6579	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6580	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6581	break;
6582
6583	default:
6584	{
6585	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6586	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6587	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6588	}
6589	}
6590	}
6591
6592	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6593	return VINF_SUCCESS;
6594	}
6595
6596
6597	/**
6598	* Saves the guest control registers from the current VMCS into the guest-CPU
6599	* context.
6600	*
6601	* @returns VBox status code.
6602	* @param pVCpu The cross context virtual CPU structure.
6603	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6604	* out-of-sync. Make sure to update the required fields
6605	* before using them.
6606	*
6607	* @remarks No-long-jump zone!!!
6608	*/
6609	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6610	{
6611	/* Guest CR0. Guest FPU. */
6612	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6613	AssertRCReturn(rc, rc);
6614
6615	/* Guest CR4. */
6616	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6617	AssertRCReturn(rc, rc);
6618
6619	/* Guest CR2 - updated always during the world-switch or in #PF. */
6620	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6621	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6622	{
6623	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6624	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6625	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6626
6627	PVM pVM = pVCpu->CTX_SUFF(pVM);
6628	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6629	\|\| ( pVM->hm.s.fNestedPaging
6630	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6631	{
6632	uint64_t u64Val = 0;
6633	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6634	if (pMixedCtx->cr3 != u64Val)
6635	{
6636	CPUMSetGuestCR3(pVCpu, u64Val);
6637	if (VMMRZCallRing3IsEnabled(pVCpu))
6638	{
6639	PGMUpdateCR3(pVCpu, u64Val);
6640	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6641	}
6642	else
6643	{
6644	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6645	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6646	}
6647	}
6648
6649	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6650	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6651	{
6652	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6653	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6654	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6655	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6656	AssertRCReturn(rc, rc);
6657
6658	if (VMMRZCallRing3IsEnabled(pVCpu))
6659	{
6660	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6661	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6662	}
6663	else
6664	{
6665	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6666	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6667	}
6668	}
6669	}
6670
6671	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6672	}
6673
6674	/*
6675	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6676	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6677	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6678	*
6679	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6680	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6681	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6682	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6683	*
6684	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6685	*/
6686	if (VMMRZCallRing3IsEnabled(pVCpu))
6687	{
6688	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6689	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6690
6691	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6692	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6693
6694	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6695	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6696	}
6697
6698	return rc;
6699	}
6700
6701
6702	/**
6703	* Reads a guest segment register from the current VMCS into the guest-CPU
6704	* context.
6705	*
6706	* @returns VBox status code.
6707	* @param pVCpu The cross context virtual CPU structure.
6708	* @param idxSel Index of the selector in the VMCS.
6709	* @param idxLimit Index of the segment limit in the VMCS.
6710	* @param idxBase Index of the segment base in the VMCS.
6711	* @param idxAccess Index of the access rights of the segment in the VMCS.
6712	* @param pSelReg Pointer to the segment selector.
6713	*
6714	* @remarks No-long-jump zone!!!
6715	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6716	* macro as that takes care of whether to read from the VMCS cache or
6717	* not.
6718	*/
6719	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6720	PCPUMSELREG pSelReg)
6721	{
6722	NOREF(pVCpu);
6723
6724	uint32_t u32Val = 0;
6725	int rc = VMXReadVmcs32(idxSel, &u32Val);
6726	AssertRCReturn(rc, rc);
6727	pSelReg->Sel = (uint16_t)u32Val;
6728	pSelReg->ValidSel = (uint16_t)u32Val;
6729	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6730
6731	rc = VMXReadVmcs32(idxLimit, &u32Val);
6732	AssertRCReturn(rc, rc);
6733	pSelReg->u32Limit = u32Val;
6734
6735	uint64_t u64Val = 0;
6736	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6737	AssertRCReturn(rc, rc);
6738	pSelReg->u64Base = u64Val;
6739
6740	rc = VMXReadVmcs32(idxAccess, &u32Val);
6741	AssertRCReturn(rc, rc);
6742	pSelReg->Attr.u = u32Val;
6743
6744	/*
6745	* If VT-x marks the segment as unusable, most other bits remain undefined:
6746	* - For CS the L, D and G bits have meaning.
6747	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6748	* - For the remaining data segments no bits are defined.
6749	*
6750	* The present bit and the unusable bit has been observed to be set at the
6751	* same time (the selector was supposed to be invalid as we started executing
6752	* a V8086 interrupt in ring-0).
6753	*
6754	* What should be important for the rest of the VBox code, is that the P bit is
6755	* cleared. Some of the other VBox code recognizes the unusable bit, but
6756	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6757	* safe side here, we'll strip off P and other bits we don't care about. If
6758	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6759	*
6760	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6761	*/
6762	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6763	{
6764	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6765
6766	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6767	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6768	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6769
6770	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6771	#ifdef DEBUG_bird
6772	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6773	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6774	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6775	#endif
6776	}
6777	return VINF_SUCCESS;
6778	}
6779
6780
6781	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6782	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6783	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6784	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6785	#else
6786	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6787	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6788	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6789	#endif
6790
6791
6792	/**
6793	* Saves the guest segment registers from the current VMCS into the guest-CPU
6794	* context.
6795	*
6796	* @returns VBox status code.
6797	* @param pVCpu The cross context virtual CPU structure.
6798	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6799	* out-of-sync. Make sure to update the required fields
6800	* before using them.
6801	*
6802	* @remarks No-long-jump zone!!!
6803	*/
6804	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6805	{
6806	/* Guest segment registers. */
6807	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6808	{
6809	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6810	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6811	AssertRCReturn(rc, rc);
6812
6813	rc = VMXLOCAL_READ_SEG(CS, cs);
6814	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6815	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6816	rc \|= VMXLOCAL_READ_SEG(ES, es);
6817	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6818	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6819	AssertRCReturn(rc, rc);
6820
6821	/* Restore segment attributes for real-on-v86 mode hack. */
6822	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6823	{
6824	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6825	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6826	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6827	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6828	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6829	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6830	}
6831	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6832	}
6833
6834	return VINF_SUCCESS;
6835	}
6836
6837
6838	/**
6839	* Saves the guest descriptor table registers and task register from the current
6840	* VMCS into the guest-CPU context.
6841	*
6842	* @returns VBox status code.
6843	* @param pVCpu The cross context virtual CPU structure.
6844	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6845	* out-of-sync. Make sure to update the required fields
6846	* before using them.
6847	*
6848	* @remarks No-long-jump zone!!!
6849	*/
6850	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6851	{
6852	int rc = VINF_SUCCESS;
6853
6854	/* Guest LDTR. */
6855	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6856	{
6857	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6858	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6859	AssertRCReturn(rc, rc);
6860	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6861	}
6862
6863	/* Guest GDTR. */
6864	uint64_t u64Val = 0;
6865	uint32_t u32Val = 0;
6866	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6867	{
6868	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6869	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6870	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6871	pMixedCtx->gdtr.pGdt = u64Val;
6872	pMixedCtx->gdtr.cbGdt = u32Val;
6873	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6874	}
6875
6876	/* Guest IDTR. */
6877	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6878	{
6879	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6880	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6881	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6882	pMixedCtx->idtr.pIdt = u64Val;
6883	pMixedCtx->idtr.cbIdt = u32Val;
6884	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6885	}
6886
6887	/* Guest TR. */
6888	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6889	{
6890	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6891	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6892	AssertRCReturn(rc, rc);
6893
6894	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6895	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6896	{
6897	rc = VMXLOCAL_READ_SEG(TR, tr);
6898	AssertRCReturn(rc, rc);
6899	}
6900	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6901	}
6902	return rc;
6903	}
6904
6905	#undef VMXLOCAL_READ_SEG
6906
6907
6908	/**
6909	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6910	* context.
6911	*
6912	* @returns VBox status code.
6913	* @param pVCpu The cross context virtual CPU structure.
6914	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6915	* out-of-sync. Make sure to update the required fields
6916	* before using them.
6917	*
6918	* @remarks No-long-jump zone!!!
6919	*/
6920	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6921	{
6922	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6923	{
6924	if (!pVCpu->hm.s.fUsingHyperDR7)
6925	{
6926	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6927	uint32_t u32Val;
6928	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6929	pMixedCtx->dr[7] = u32Val;
6930	}
6931
6932	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6933	}
6934	return VINF_SUCCESS;
6935	}
6936
6937
6938	/**
6939	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6940	*
6941	* @returns VBox status code.
6942	* @param pVCpu The cross context virtual CPU structure.
6943	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6944	* out-of-sync. Make sure to update the required fields
6945	* before using them.
6946	*
6947	* @remarks No-long-jump zone!!!
6948	*/
6949	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6950	{
6951	NOREF(pMixedCtx);
6952
6953	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6954	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6955	return VINF_SUCCESS;
6956	}
6957
6958
6959	/**
6960	* Saves the entire guest state from the currently active VMCS into the
6961	* guest-CPU context.
6962	*
6963	* This essentially VMREADs all guest-data.
6964	*
6965	* @returns VBox status code.
6966	* @param pVCpu The cross context virtual CPU structure.
6967	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6968	* out-of-sync. Make sure to update the required fields
6969	* before using them.
6970	*/
6971	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6972	{
6973	Assert(pVCpu);
6974	Assert(pMixedCtx);
6975
6976	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6977	return VINF_SUCCESS;
6978
6979	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6980	again on the ring-3 callback path, there is no real need to. */
6981	if (VMMRZCallRing3IsEnabled(pVCpu))
6982	VMMR0LogFlushDisable(pVCpu);
6983	else
6984	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6985	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6986
6987	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6988	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6989
6990	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6991	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6992
6993	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6994	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6995
6996	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6997	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6998
6999	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7000	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7001
7002	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7003	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7004
7005	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7006	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7007
7008	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7009	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7010
7011	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7012	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7013
7014	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7015	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7016
7017	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7018	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7019	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7020
7021	if (VMMRZCallRing3IsEnabled(pVCpu))
7022	VMMR0LogFlushEnable(pVCpu);
7023
7024	return VINF_SUCCESS;
7025	}
7026
7027
7028	/**
7029	* Saves basic guest registers needed for IEM instruction execution.
7030	*
7031	* @returns VBox status code (OR-able).
7032	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7033	* @param pMixedCtx Pointer to the CPU context of the guest.
7034	* @param fMemory Whether the instruction being executed operates on
7035	* memory or not. Only CR0 is synced up if clear.
7036	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7037	*/
7038	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7039	{
7040	/*
7041	* We assume all general purpose registers other than RSP are available.
7042	*
7043	* - RIP is a must, as it will be incremented or otherwise changed.
7044	* - RFLAGS are always required to figure the CPL.
7045	* - RSP isn't always required, however it's a GPR, so frequently required.
7046	* - SS and CS are the only segment register needed if IEM doesn't do memory
7047	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7048	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7049	* be required for memory accesses.
7050	*
7051	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7052	*/
7053	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7054	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7055	if (fNeedRsp)
7056	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7057	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7058	if (!fMemory)
7059	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7060	else
7061	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7062	AssertRCReturn(rc, rc);
7063	return rc;
7064	}
7065
7066
7067	/**
7068	* Ensures that we've got a complete basic guest-context.
7069	*
7070	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7071	* is for the interpreter.
7072	*
7073	* @returns VBox status code.
7074	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7075	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7076	* needing to be synced in.
7077	* @thread EMT(pVCpu)
7078	*/
7079	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7080	{
7081	/* Note! Since this is only applicable to VT-x, the implementation is placed
7082	in the VT-x part of the sources instead of the generic stuff. */
7083	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7084	{
7085	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7086	/*
7087	* For now, imply that the caller might change everything too. Do this after
7088	* saving the guest state so as to not trigger assertions.
7089	*/
7090	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7091	return rc;
7092	}
7093	return VINF_SUCCESS;
7094	}
7095
7096
7097	/**
7098	* Check per-VM and per-VCPU force flag actions that require us to go back to
7099	* ring-3 for one reason or another.
7100	*
7101	* @returns Strict VBox status code (i.e. informational status codes too)
7102	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7103	* ring-3.
7104	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7105	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7106	* interrupts)
7107	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7108	* all EMTs to be in ring-3.
7109	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7110	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7111	* to the EM loop.
7112	*
7113	* @param pVM The cross context VM structure.
7114	* @param pVCpu The cross context virtual CPU structure.
7115	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7116	* out-of-sync. Make sure to update the required fields
7117	* before using them.
7118	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7119	*/
7120	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7121	{
7122	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7123
7124	/*
7125	* Anything pending? Should be more likely than not if we're doing a good job.
7126	*/
7127	if ( !fStepping
7128	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7129	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7130	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7131	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7132	return VINF_SUCCESS;
7133
7134	/* We need the control registers now, make sure the guest-CPU context is updated. */
7135	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7136	AssertRCReturn(rc3, rc3);
7137
7138	/* Pending HM CR3 sync. */
7139	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7140	{
7141	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7142	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7143	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7144	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7145	}
7146
7147	/* Pending HM PAE PDPEs. */
7148	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7149	{
7150	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7151	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7152	}
7153
7154	/* Pending PGM C3 sync. */
7155	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7156	{
7157	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7158	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7159	if (rcStrict2 != VINF_SUCCESS)
7160	{
7161	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7162	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7163	return rcStrict2;
7164	}
7165	}
7166
7167	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7168	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7169	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7170	{
7171	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7172	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7173	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7174	return rc2;
7175	}
7176
7177	/* Pending VM request packets, such as hardware interrupts. */
7178	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7179	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7180	{
7181	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7182	return VINF_EM_PENDING_REQUEST;
7183	}
7184
7185	/* Pending PGM pool flushes. */
7186	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7187	{
7188	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7189	return VINF_PGM_POOL_FLUSH_PENDING;
7190	}
7191
7192	/* Pending DMA requests. */
7193	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7194	{
7195	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7196	return VINF_EM_RAW_TO_R3;
7197	}
7198
7199	return VINF_SUCCESS;
7200	}
7201
7202
7203	/**
7204	* Converts any TRPM trap into a pending HM event. This is typically used when
7205	* entering from ring-3 (not longjmp returns).
7206	*
7207	* @param pVCpu The cross context virtual CPU structure.
7208	*/
7209	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7210	{
7211	Assert(TRPMHasTrap(pVCpu));
7212	Assert(!pVCpu->hm.s.Event.fPending);
7213
7214	uint8_t uVector;
7215	TRPMEVENT enmTrpmEvent;
7216	RTGCUINT uErrCode;
7217	RTGCUINTPTR GCPtrFaultAddress;
7218	uint8_t cbInstr;
7219
7220	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7221	AssertRC(rc);
7222
7223	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7224	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7225	if (enmTrpmEvent == TRPM_TRAP)
7226	{
7227	switch (uVector)
7228	{
7229	case X86_XCPT_NMI:
7230	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7231	break;
7232
7233	case X86_XCPT_BP:
7234	case X86_XCPT_OF:
7235	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7236	break;
7237
7238	case X86_XCPT_PF:
7239	case X86_XCPT_DF:
7240	case X86_XCPT_TS:
7241	case X86_XCPT_NP:
7242	case X86_XCPT_SS:
7243	case X86_XCPT_GP:
7244	case X86_XCPT_AC:
7245	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7246	/* fall thru */
7247	default:
7248	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7249	break;
7250	}
7251	}
7252	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7253	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7254	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7255	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7256	else
7257	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7258
7259	rc = TRPMResetTrap(pVCpu);
7260	AssertRC(rc);
7261	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7262	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7263
7264	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7265	}
7266
7267
7268	/**
7269	* Converts the pending HM event into a TRPM trap.
7270	*
7271	* @param pVCpu The cross context virtual CPU structure.
7272	*/
7273	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7274	{
7275	Assert(pVCpu->hm.s.Event.fPending);
7276
7277	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7278	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7279	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7280	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7281
7282	/* If a trap was already pending, we did something wrong! */
7283	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7284
7285	TRPMEVENT enmTrapType;
7286	switch (uVectorType)
7287	{
7288	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7289	enmTrapType = TRPM_HARDWARE_INT;
7290	break;
7291
7292	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7293	enmTrapType = TRPM_SOFTWARE_INT;
7294	break;
7295
7296	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7297	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7298	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7299	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7300	enmTrapType = TRPM_TRAP;
7301	break;
7302
7303	default:
7304	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7305	enmTrapType = TRPM_32BIT_HACK;
7306	break;
7307	}
7308
7309	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7310
7311	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7312	AssertRC(rc);
7313
7314	if (fErrorCodeValid)
7315	TRPMSetErrorCode(pVCpu, uErrorCode);
7316
7317	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7318	&& uVector == X86_XCPT_PF)
7319	{
7320	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7321	}
7322	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7323	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7324	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7325	{
7326	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7327	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7328	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7329	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7330	}
7331
7332	/* Clear any pending events from the VMCS. */
7333	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7334	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7335
7336	/* We're now done converting the pending event. */
7337	pVCpu->hm.s.Event.fPending = false;
7338	}
7339
7340
7341	/**
7342	* Does the necessary state syncing before returning to ring-3 for any reason
7343	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7344	*
7345	* @returns VBox status code.
7346	* @param pVCpu The cross context virtual CPU structure.
7347	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7348	* be out-of-sync. Make sure to update the required
7349	* fields before using them.
7350	* @param fSaveGuestState Whether to save the guest state or not.
7351	*
7352	* @remarks No-long-jmp zone!!!
7353	*/
7354	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7355	{
7356	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7357	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7358
7359	RTCPUID idCpu = RTMpCpuId();
7360	Log4Func(("HostCpuId=%u\n", idCpu));
7361
7362	/*
7363	* !!! IMPORTANT !!!
7364	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7365	*/
7366
7367	/* Save the guest state if necessary. */
7368	if ( fSaveGuestState
7369	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7370	{
7371	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7372	AssertRCReturn(rc, rc);
7373	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7374	}
7375
7376	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7377	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7378	{
7379	/* We shouldn't reload CR0 without saving it first. */
7380	if (!fSaveGuestState)
7381	{
7382	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7383	AssertRCReturn(rc, rc);
7384	}
7385	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7386	}
7387
7388	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7389	#ifdef VBOX_STRICT
7390	if (CPUMIsHyperDebugStateActive(pVCpu))
7391	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7392	#endif
7393	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7394	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7395	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7396	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7397
7398	#if HC_ARCH_BITS == 64
7399	/* Restore host-state bits that VT-x only restores partially. */
7400	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7401	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7402	{
7403	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7404	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7405	}
7406	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7407	#endif
7408
7409	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7410	if (pVCpu->hm.s.vmx.fLazyMsrs)
7411	{
7412	/* We shouldn't reload the guest MSRs without saving it first. */
7413	if (!fSaveGuestState)
7414	{
7415	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7416	AssertRCReturn(rc, rc);
7417	}
7418	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7419	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7420	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7421	}
7422
7423	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7424	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7425
7426	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7427	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7428	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7429	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7430	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7431	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7432	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7433	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7434
7435	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7436
7437	/** @todo This partially defeats the purpose of having preemption hooks.
7438	* The problem is, deregistering the hooks should be moved to a place that
7439	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7440	* context.
7441	*/
7442	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7443	{
7444	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7445	AssertRCReturn(rc, rc);
7446
7447	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7448	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7449	}
7450	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7451	NOREF(idCpu);
7452
7453	return VINF_SUCCESS;
7454	}
7455
7456
7457	/**
7458	* Leaves the VT-x session.
7459	*
7460	* @returns VBox status code.
7461	* @param pVCpu The cross context virtual CPU structure.
7462	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7463	* out-of-sync. Make sure to update the required fields
7464	* before using them.
7465	*
7466	* @remarks No-long-jmp zone!!!
7467	*/
7468	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7469	{
7470	HM_DISABLE_PREEMPT();
7471	HMVMX_ASSERT_CPU_SAFE();
7472	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7473	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7474
7475	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7476	and done this from the VMXR0ThreadCtxCallback(). */
7477	if (!pVCpu->hm.s.fLeaveDone)
7478	{
7479	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7480	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7481	pVCpu->hm.s.fLeaveDone = true;
7482	}
7483	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7484
7485	/*
7486	* !!! IMPORTANT !!!
7487	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7488	*/
7489
7490	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7491	/** @todo Deregistering here means we need to VMCLEAR always
7492	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7493	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7494	VMMR0ThreadCtxHookDisable(pVCpu);
7495
7496	/* Leave HM context. This takes care of local init (term). */
7497	int rc = HMR0LeaveCpu(pVCpu);
7498
7499	HM_RESTORE_PREEMPT();
7500	return rc;
7501	}
7502
7503
7504	/**
7505	* Does the necessary state syncing before doing a longjmp to ring-3.
7506	*
7507	* @returns VBox status code.
7508	* @param pVCpu The cross context virtual CPU structure.
7509	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7510	* out-of-sync. Make sure to update the required fields
7511	* before using them.
7512	*
7513	* @remarks No-long-jmp zone!!!
7514	*/
7515	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7516	{
7517	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7518	}
7519
7520
7521	/**
7522	* Take necessary actions before going back to ring-3.
7523	*
7524	* An action requires us to go back to ring-3. This function does the necessary
7525	* steps before we can safely return to ring-3. This is not the same as longjmps
7526	* to ring-3, this is voluntary and prepares the guest so it may continue
7527	* executing outside HM (recompiler/IEM).
7528	*
7529	* @returns VBox status code.
7530	* @param pVM The cross context VM structure.
7531	* @param pVCpu The cross context virtual CPU structure.
7532	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7533	* out-of-sync. Make sure to update the required fields
7534	* before using them.
7535	* @param rcExit The reason for exiting to ring-3. Can be
7536	* VINF_VMM_UNKNOWN_RING3_CALL.
7537	*/
7538	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7539	{
7540	Assert(pVM);
7541	Assert(pVCpu);
7542	Assert(pMixedCtx);
7543	HMVMX_ASSERT_PREEMPT_SAFE();
7544
7545	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7546	{
7547	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7548	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7549	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7550	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7551	}
7552
7553	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7554	VMMRZCallRing3Disable(pVCpu);
7555	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7556
7557	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7558	if (pVCpu->hm.s.Event.fPending)
7559	{
7560	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7561	Assert(!pVCpu->hm.s.Event.fPending);
7562	}
7563
7564	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7565	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7566
7567	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7568	and if we're injecting an event we should have a TRPM trap pending. */
7569	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7570	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7571	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7572	#endif
7573
7574	/* Save guest state and restore host state bits. */
7575	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7576	AssertRCReturn(rc, rc);
7577	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7578	/* Thread-context hooks are unregistered at this point!!! */
7579
7580	/* Sync recompiler state. */
7581	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7582	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7583	\| CPUM_CHANGED_LDTR
7584	\| CPUM_CHANGED_GDTR
7585	\| CPUM_CHANGED_IDTR
7586	\| CPUM_CHANGED_TR
7587	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7588	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7589	if ( pVM->hm.s.fNestedPaging
7590	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7591	{
7592	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7593	}
7594
7595	Assert(!pVCpu->hm.s.fClearTrapFlag);
7596
7597	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7598	if (rcExit != VINF_EM_RAW_INTERRUPT)
7599	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7600
7601	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7602
7603	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7604	VMMRZCallRing3RemoveNotification(pVCpu);
7605	VMMRZCallRing3Enable(pVCpu);
7606
7607	return rc;
7608	}
7609
7610
7611	/**
7612	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7613	* longjump to ring-3 and possibly get preempted.
7614	*
7615	* @returns VBox status code.
7616	* @param pVCpu The cross context virtual CPU structure.
7617	* @param enmOperation The operation causing the ring-3 longjump.
7618	* @param pvUser Opaque pointer to the guest-CPU context. The data
7619	* may be out-of-sync. Make sure to update the required
7620	* fields before using them.
7621	*/
7622	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7623	{
7624	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7625	{
7626	/*
7627	* !!! IMPORTANT !!!
7628	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7629	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7630	*/
7631	VMMRZCallRing3RemoveNotification(pVCpu);
7632	VMMRZCallRing3Disable(pVCpu);
7633	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7634	RTThreadPreemptDisable(&PreemptState);
7635
7636	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7637	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7638
7639	#if HC_ARCH_BITS == 64
7640	/* Restore host-state bits that VT-x only restores partially. */
7641	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7642	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7643	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7644	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7645	#endif
7646	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7647	if (pVCpu->hm.s.vmx.fLazyMsrs)
7648	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7649
7650	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7651	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7652	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7653	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7654	{
7655	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7656	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7657	}
7658
7659	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7660	VMMR0ThreadCtxHookDisable(pVCpu);
7661	HMR0LeaveCpu(pVCpu);
7662	RTThreadPreemptRestore(&PreemptState);
7663	return VINF_SUCCESS;
7664	}
7665
7666	Assert(pVCpu);
7667	Assert(pvUser);
7668	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7669	HMVMX_ASSERT_PREEMPT_SAFE();
7670
7671	VMMRZCallRing3Disable(pVCpu);
7672	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7673
7674	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7675	enmOperation));
7676
7677	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7678	AssertRCReturn(rc, rc);
7679
7680	VMMRZCallRing3Enable(pVCpu);
7681	return VINF_SUCCESS;
7682	}
7683
7684
7685	/**
7686	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7687	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7688	*
7689	* @param pVCpu The cross context virtual CPU structure.
7690	*/
7691	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7692	{
7693	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7694	{
7695	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7696	{
7697	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7698	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7699	AssertRC(rc);
7700	Log4(("Setup interrupt-window exiting\n"));
7701	}
7702	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7703	}
7704
7705
7706	/**
7707	* Clears the interrupt-window exiting control in the VMCS.
7708	*
7709	* @param pVCpu The cross context virtual CPU structure.
7710	*/
7711	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7712	{
7713	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7714	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7715	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7716	AssertRC(rc);
7717	Log4(("Cleared interrupt-window exiting\n"));
7718	}
7719
7720
7721	/**
7722	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7723	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7724	*
7725	* @param pVCpu The cross context virtual CPU structure.
7726	*/
7727	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7728	{
7729	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7730	{
7731	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7732	{
7733	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7734	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7735	AssertRC(rc);
7736	Log4(("Setup NMI-window exiting\n"));
7737	}
7738	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7739	}
7740
7741
7742	/**
7743	* Clears the NMI-window exiting control in the VMCS.
7744	*
7745	* @param pVCpu The cross context virtual CPU structure.
7746	*/
7747	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7748	{
7749	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7750	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7751	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7752	AssertRC(rc);
7753	Log4(("Cleared NMI-window exiting\n"));
7754	}
7755
7756
7757	/**
7758	* Evaluates the event to be delivered to the guest and sets it as the pending
7759	* event.
7760	*
7761	* @returns The VT-x guest-interruptibility state.
7762	* @param pVCpu The cross context virtual CPU structure.
7763	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7764	* out-of-sync. Make sure to update the required fields
7765	* before using them.
7766	*/
7767	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7768	{
7769	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7770	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7771	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7772	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7773	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7774
7775	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7776	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7777	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7778	Assert(!TRPMHasTrap(pVCpu));
7779
7780	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7781	APICUpdatePendingInterrupts(pVCpu);
7782
7783	/*
7784	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7785	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7786	*/
7787	/** @todo SMI. SMIs take priority over NMIs. */
7788	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7789	{
7790	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7791	if ( !pVCpu->hm.s.Event.fPending
7792	&& !fBlockNmi
7793	&& !fBlockSti
7794	&& !fBlockMovSS)
7795	{
7796	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7797	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7798	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7799
7800	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7801	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7802	}
7803	else
7804	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7805	}
7806	/*
7807	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7808	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7809	*/
7810	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7811	&& !pVCpu->hm.s.fSingleInstruction)
7812	{
7813	Assert(!DBGFIsStepping(pVCpu));
7814	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7815	AssertRC(rc);
7816	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7817	if ( !pVCpu->hm.s.Event.fPending
7818	&& !fBlockInt
7819	&& !fBlockSti
7820	&& !fBlockMovSS)
7821	{
7822	uint8_t u8Interrupt;
7823	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7824	if (RT_SUCCESS(rc))
7825	{
7826	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7827	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7828	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7829
7830	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7831	}
7832	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7833	{
7834	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7835	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7836	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7837
7838	/*
7839	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7840	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7841	* need to re-set this force-flag here.
7842	*/
7843	}
7844	else
7845	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7846	}
7847	else
7848	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7849	}
7850
7851	return uIntrState;
7852	}
7853
7854
7855	/**
7856	* Sets a pending-debug exception to be delivered to the guest if the guest is
7857	* single-stepping in the VMCS.
7858	*
7859	* @param pVCpu The cross context virtual CPU structure.
7860	*/
7861	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7862	{
7863	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7864	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7865	AssertRC(rc);
7866	}
7867
7868
7869	/**
7870	* Injects any pending events into the guest if the guest is in a state to
7871	* receive them.
7872	*
7873	* @returns Strict VBox status code (i.e. informational status codes too).
7874	* @param pVCpu The cross context virtual CPU structure.
7875	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7876	* out-of-sync. Make sure to update the required fields
7877	* before using them.
7878	* @param uIntrState The VT-x guest-interruptibility state.
7879	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7880	* return VINF_EM_DBG_STEPPED if the event was
7881	* dispatched directly.
7882	*/
7883	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7884	{
7885	HMVMX_ASSERT_PREEMPT_SAFE();
7886	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7887
7888	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7889	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7890
7891	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7892	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7893	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7894	Assert(!TRPMHasTrap(pVCpu));
7895
7896	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7897	if (pVCpu->hm.s.Event.fPending)
7898	{
7899	/*
7900	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7901	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7902	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7903	*
7904	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7905	*/
7906	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7907	#ifdef VBOX_STRICT
7908	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7909	{
7910	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7911	Assert(!fBlockInt);
7912	Assert(!fBlockSti);
7913	Assert(!fBlockMovSS);
7914	}
7915	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7916	{
7917	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7918	Assert(!fBlockSti);
7919	Assert(!fBlockMovSS);
7920	Assert(!fBlockNmi);
7921	}
7922	#endif
7923	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7924	(uint8_t)uIntType));
7925	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7926	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7927	fStepping, &uIntrState);
7928	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7929
7930	/* Update the interruptibility-state as it could have been changed by
7931	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7932	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7933	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7934
7935	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7936	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7937	else
7938	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7939	}
7940
7941	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7942	if ( fBlockSti
7943	\|\| fBlockMovSS)
7944	{
7945	if (!pVCpu->hm.s.fSingleInstruction)
7946	{
7947	/*
7948	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7949	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7950	* See Intel spec. 27.3.4 "Saving Non-Register State".
7951	*/
7952	Assert(!DBGFIsStepping(pVCpu));
7953	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7954	AssertRCReturn(rc2, rc2);
7955	if (pMixedCtx->eflags.Bits.u1TF)
7956	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7957	}
7958	else if (pMixedCtx->eflags.Bits.u1TF)
7959	{
7960	/*
7961	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7962	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7963	*/
7964	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7965	uIntrState = 0;
7966	}
7967	}
7968
7969	/*
7970	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7971	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7972	*/
7973	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7974	AssertRC(rc2);
7975
7976	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7977	NOREF(fBlockMovSS); NOREF(fBlockSti);
7978	return rcStrict;
7979	}
7980
7981
7982	/**
7983	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7984	*
7985	* @param pVCpu The cross context virtual CPU structure.
7986	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7987	* out-of-sync. Make sure to update the required fields
7988	* before using them.
7989	*/
7990	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7991	{
7992	NOREF(pMixedCtx);
7993	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7994	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7995	}
7996
7997
7998	/**
7999	* Injects a double-fault (\#DF) exception into the VM.
8000	*
8001	* @returns Strict VBox status code (i.e. informational status codes too).
8002	* @param pVCpu The cross context virtual CPU structure.
8003	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8004	* out-of-sync. Make sure to update the required fields
8005	* before using them.
8006	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8007	* and should return VINF_EM_DBG_STEPPED if the event
8008	* is injected directly (register modified by us, not
8009	* by hardware on VM-entry).
8010	* @param puIntrState Pointer to the current guest interruptibility-state.
8011	* This interruptibility-state will be updated if
8012	* necessary. This cannot not be NULL.
8013	*/
8014	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8015	{
8016	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8017	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8018	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8019	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8020	fStepping, puIntrState);
8021	}
8022
8023
8024	/**
8025	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8026	*
8027	* @param pVCpu The cross context virtual CPU structure.
8028	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8029	* out-of-sync. Make sure to update the required fields
8030	* before using them.
8031	*/
8032	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8033	{
8034	NOREF(pMixedCtx);
8035	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8036	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8037	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8038	}
8039
8040
8041	/**
8042	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8043	*
8044	* @param pVCpu The cross context virtual CPU structure.
8045	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8046	* out-of-sync. Make sure to update the required fields
8047	* before using them.
8048	* @param cbInstr The value of RIP that is to be pushed on the guest
8049	* stack.
8050	*/
8051	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8052	{
8053	NOREF(pMixedCtx);
8054	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8055	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8056	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8057	}
8058
8059
8060	/**
8061	* Injects a general-protection (\#GP) fault into the VM.
8062	*
8063	* @returns Strict VBox status code (i.e. informational status codes too).
8064	* @param pVCpu The cross context virtual CPU structure.
8065	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8066	* out-of-sync. Make sure to update the required fields
8067	* before using them.
8068	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8069	* mode, i.e. in real-mode it's not valid).
8070	* @param u32ErrorCode The error code associated with the \#GP.
8071	* @param fStepping Whether we're running in
8072	* hmR0VmxRunGuestCodeStep() and should return
8073	* VINF_EM_DBG_STEPPED if the event is injected
8074	* directly (register modified by us, not by
8075	* hardware on VM-entry).
8076	* @param puIntrState Pointer to the current guest interruptibility-state.
8077	* This interruptibility-state will be updated if
8078	* necessary. This cannot not be NULL.
8079	*/
8080	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8081	bool fStepping, uint32_t *puIntrState)
8082	{
8083	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8084	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8085	if (fErrorCodeValid)
8086	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8087	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8088	fStepping, puIntrState);
8089	}
8090
8091
8092	#if 0 /* unused */
8093	/**
8094	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8095	* VM.
8096	*
8097	* @param pVCpu The cross context virtual CPU structure.
8098	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8099	* out-of-sync. Make sure to update the required fields
8100	* before using them.
8101	* @param u32ErrorCode The error code associated with the \#GP.
8102	*/
8103	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8104	{
8105	NOREF(pMixedCtx);
8106	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8107	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8108	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8109	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8110	}
8111	#endif /* unused */
8112
8113
8114	/**
8115	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8116	*
8117	* @param pVCpu The cross context virtual CPU structure.
8118	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8119	* out-of-sync. Make sure to update the required fields
8120	* before using them.
8121	* @param uVector The software interrupt vector number.
8122	* @param cbInstr The value of RIP that is to be pushed on the guest
8123	* stack.
8124	*/
8125	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8126	{
8127	NOREF(pMixedCtx);
8128	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8129	if ( uVector == X86_XCPT_BP
8130	\|\| uVector == X86_XCPT_OF)
8131	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8132	else
8133	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8134	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8135	}
8136
8137
8138	/**
8139	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8140	* stack.
8141	*
8142	* @returns Strict VBox status code (i.e. informational status codes too).
8143	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8144	* @param pVM The cross context VM structure.
8145	* @param pMixedCtx Pointer to the guest-CPU context.
8146	* @param uValue The value to push to the guest stack.
8147	*/
8148	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8149	{
8150	/*
8151	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8152	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8153	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8154	*/
8155	if (pMixedCtx->sp == 1)
8156	return VINF_EM_RESET;
8157	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8158	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8159	AssertRC(rc);
8160	return rc;
8161	}
8162
8163
8164	/**
8165	* Injects an event into the guest upon VM-entry by updating the relevant fields
8166	* in the VM-entry area in the VMCS.
8167	*
8168	* @returns Strict VBox status code (i.e. informational status codes too).
8169	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8170	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8171	*
8172	* @param pVCpu The cross context virtual CPU structure.
8173	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8174	* be out-of-sync. Make sure to update the required
8175	* fields before using them.
8176	* @param u64IntInfo The VM-entry interruption-information field.
8177	* @param cbInstr The VM-entry instruction length in bytes (for
8178	* software interrupts, exceptions and privileged
8179	* software exceptions).
8180	* @param u32ErrCode The VM-entry exception error code.
8181	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8182	* @param puIntrState Pointer to the current guest interruptibility-state.
8183	* This interruptibility-state will be updated if
8184	* necessary. This cannot not be NULL.
8185	* @param fStepping Whether we're running in
8186	* hmR0VmxRunGuestCodeStep() and should return
8187	* VINF_EM_DBG_STEPPED if the event is injected
8188	* directly (register modified by us, not by
8189	* hardware on VM-entry).
8190	*
8191	* @remarks Requires CR0!
8192	*/
8193	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8194	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8195	uint32_t *puIntrState)
8196	{
8197	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8198	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8199	Assert(puIntrState);
8200	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8201
8202	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8203	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8204
8205	#ifdef VBOX_STRICT
8206	/* Validate the error-code-valid bit for hardware exceptions. */
8207	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8208	{
8209	switch (uVector)
8210	{
8211	case X86_XCPT_PF:
8212	case X86_XCPT_DF:
8213	case X86_XCPT_TS:
8214	case X86_XCPT_NP:
8215	case X86_XCPT_SS:
8216	case X86_XCPT_GP:
8217	case X86_XCPT_AC:
8218	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8219	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8220	/* fall thru */
8221	default:
8222	break;
8223	}
8224	}
8225	#endif
8226
8227	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8228	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8229	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8230
8231	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8232
8233	/* We require CR0 to check if the guest is in real-mode. */
8234	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8235	AssertRCReturn(rc, rc);
8236
8237	/*
8238	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8239	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8240	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8241	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8242	*/
8243	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8244	{
8245	PVM pVM = pVCpu->CTX_SUFF(pVM);
8246	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8247	{
8248	Assert(PDMVmmDevHeapIsEnabled(pVM));
8249	Assert(pVM->hm.s.vmx.pRealModeTSS);
8250
8251	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8252	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8253	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8254	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8255	AssertRCReturn(rc, rc);
8256	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8257
8258	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8259	size_t const cbIdtEntry = sizeof(X86IDTR16);
8260	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8261	{
8262	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8263	if (uVector == X86_XCPT_DF)
8264	return VINF_EM_RESET;
8265
8266	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8267	if (uVector == X86_XCPT_GP)
8268	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8269
8270	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8271	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8272	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8273	fStepping, puIntrState);
8274	}
8275
8276	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8277	uint16_t uGuestIp = pMixedCtx->ip;
8278	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8279	{
8280	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8281	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8282	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8283	}
8284	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8285	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8286
8287	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8288	X86IDTR16 IdtEntry;
8289	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8290	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8291	AssertRCReturn(rc, rc);
8292
8293	/* Construct the stack frame for the interrupt/exception handler. */
8294	VBOXSTRICTRC rcStrict;
8295	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8296	if (rcStrict == VINF_SUCCESS)
8297	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8298	if (rcStrict == VINF_SUCCESS)
8299	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8300
8301	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8302	if (rcStrict == VINF_SUCCESS)
8303	{
8304	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8305	pMixedCtx->rip = IdtEntry.offSel;
8306	pMixedCtx->cs.Sel = IdtEntry.uSel;
8307	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8308	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8309	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8310	&& uVector == X86_XCPT_PF)
8311	pMixedCtx->cr2 = GCPtrFaultAddress;
8312
8313	/* If any other guest-state bits are changed here, make sure to update
8314	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8315	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8316	\| HM_CHANGED_GUEST_RIP
8317	\| HM_CHANGED_GUEST_RFLAGS
8318	\| HM_CHANGED_GUEST_RSP);
8319
8320	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8321	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8322	{
8323	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8324	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8325	Log4(("Clearing inhibition due to STI.\n"));
8326	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8327	}
8328	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8329	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8330
8331	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8332	it, if we are returning to ring-3 before executing guest code. */
8333	pVCpu->hm.s.Event.fPending = false;
8334
8335	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8336	if (fStepping)
8337	rcStrict = VINF_EM_DBG_STEPPED;
8338	}
8339	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8340	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8341	return rcStrict;
8342	}
8343
8344	/*
8345	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8346	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8347	*/
8348	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8349	}
8350
8351	/* Validate. */
8352	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8353	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8354	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8355
8356	/* Inject. */
8357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8358	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8359	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8360	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8361
8362	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8363	&& uVector == X86_XCPT_PF)
8364	pMixedCtx->cr2 = GCPtrFaultAddress;
8365
8366	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8367	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8368
8369	AssertRCReturn(rc, rc);
8370	return VINF_SUCCESS;
8371	}
8372
8373
8374	/**
8375	* Clears the interrupt-window exiting control in the VMCS and if necessary
8376	* clears the current event in the VMCS as well.
8377	*
8378	* @returns VBox status code.
8379	* @param pVCpu The cross context virtual CPU structure.
8380	*
8381	* @remarks Use this function only to clear events that have not yet been
8382	* delivered to the guest but are injected in the VMCS!
8383	* @remarks No-long-jump zone!!!
8384	*/
8385	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8386	{
8387	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8388
8389	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8390	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8391
8392	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8393	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8394	}
8395
8396
8397	/**
8398	* Enters the VT-x session.
8399	*
8400	* @returns VBox status code.
8401	* @param pVM The cross context VM structure.
8402	* @param pVCpu The cross context virtual CPU structure.
8403	* @param pCpu Pointer to the CPU info struct.
8404	*/
8405	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8406	{
8407	AssertPtr(pVM);
8408	AssertPtr(pVCpu);
8409	Assert(pVM->hm.s.vmx.fSupported);
8410	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8411	NOREF(pCpu); NOREF(pVM);
8412
8413	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8414	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8415
8416	#ifdef VBOX_STRICT
8417	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8418	RTCCUINTREG uHostCR4 = ASMGetCR4();
8419	if (!(uHostCR4 & X86_CR4_VMXE))
8420	{
8421	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8422	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8423	}
8424	#endif
8425
8426	/*
8427	* Load the VCPU's VMCS as the current (and active) one.
8428	*/
8429	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8430	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8431	if (RT_FAILURE(rc))
8432	return rc;
8433
8434	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8435	pVCpu->hm.s.fLeaveDone = false;
8436	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8437
8438	return VINF_SUCCESS;
8439	}
8440
8441
8442	/**
8443	* The thread-context callback (only on platforms which support it).
8444	*
8445	* @param enmEvent The thread-context event.
8446	* @param pVCpu The cross context virtual CPU structure.
8447	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8448	* @thread EMT(pVCpu)
8449	*/
8450	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8451	{
8452	NOREF(fGlobalInit);
8453
8454	switch (enmEvent)
8455	{
8456	case RTTHREADCTXEVENT_OUT:
8457	{
8458	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8459	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8460	VMCPU_ASSERT_EMT(pVCpu);
8461
8462	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8463
8464	/* No longjmps (logger flushes, locks) in this fragile context. */
8465	VMMRZCallRing3Disable(pVCpu);
8466	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8467
8468	/*
8469	* Restore host-state (FPU, debug etc.)
8470	*/
8471	if (!pVCpu->hm.s.fLeaveDone)
8472	{
8473	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8474	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8475	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8476	pVCpu->hm.s.fLeaveDone = true;
8477	}
8478
8479	/* Leave HM context, takes care of local init (term). */
8480	int rc = HMR0LeaveCpu(pVCpu);
8481	AssertRC(rc); NOREF(rc);
8482
8483	/* Restore longjmp state. */
8484	VMMRZCallRing3Enable(pVCpu);
8485	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8486	break;
8487	}
8488
8489	case RTTHREADCTXEVENT_IN:
8490	{
8491	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8492	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8493	VMCPU_ASSERT_EMT(pVCpu);
8494
8495	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8496	VMMRZCallRing3Disable(pVCpu);
8497	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8498
8499	/* Initialize the bare minimum state required for HM. This takes care of
8500	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8501	int rc = HMR0EnterCpu(pVCpu);
8502	AssertRC(rc);
8503	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8504
8505	/* Load the active VMCS as the current one. */
8506	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8507	{
8508	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8509	AssertRC(rc); NOREF(rc);
8510	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8511	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8512	}
8513	pVCpu->hm.s.fLeaveDone = false;
8514
8515	/* Restore longjmp state. */
8516	VMMRZCallRing3Enable(pVCpu);
8517	break;
8518	}
8519
8520	default:
8521	break;
8522	}
8523	}
8524
8525
8526	/**
8527	* Saves the host state in the VMCS host-state.
8528	* Sets up the VM-exit MSR-load area.
8529	*
8530	* The CPU state will be loaded from these fields on every successful VM-exit.
8531	*
8532	* @returns VBox status code.
8533	* @param pVM The cross context VM structure.
8534	* @param pVCpu The cross context virtual CPU structure.
8535	*
8536	* @remarks No-long-jump zone!!!
8537	*/
8538	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8539	{
8540	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8541
8542	int rc = VINF_SUCCESS;
8543	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8544	{
8545	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8546	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8547
8548	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8549	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8550
8551	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8552	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8553
8554	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8555	}
8556	return rc;
8557	}
8558
8559
8560	/**
8561	* Saves the host state in the VMCS host-state.
8562	*
8563	* @returns VBox status code.
8564	* @param pVM The cross context VM structure.
8565	* @param pVCpu The cross context virtual CPU structure.
8566	*
8567	* @remarks No-long-jump zone!!!
8568	*/
8569	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8570	{
8571	AssertPtr(pVM);
8572	AssertPtr(pVCpu);
8573
8574	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8575
8576	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8577	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8578	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8579	return hmR0VmxSaveHostState(pVM, pVCpu);
8580	}
8581
8582
8583	/**
8584	* Loads the guest state into the VMCS guest-state area.
8585	*
8586	* The will typically be done before VM-entry when the guest-CPU state and the
8587	* VMCS state may potentially be out of sync.
8588	*
8589	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8590	* VM-entry controls.
8591	* Sets up the appropriate VMX non-root function to execute guest code based on
8592	* the guest CPU mode.
8593	*
8594	* @returns VBox strict status code.
8595	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8596	* without unrestricted guest access and the VMMDev is not presently
8597	* mapped (e.g. EFI32).
8598	*
8599	* @param pVM The cross context VM structure.
8600	* @param pVCpu The cross context virtual CPU structure.
8601	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8602	* out-of-sync. Make sure to update the required fields
8603	* before using them.
8604	*
8605	* @remarks No-long-jump zone!!!
8606	*/
8607	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8608	{
8609	AssertPtr(pVM);
8610	AssertPtr(pVCpu);
8611	AssertPtr(pMixedCtx);
8612	HMVMX_ASSERT_PREEMPT_SAFE();
8613
8614	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8615
8616	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8617
8618	/* Determine real-on-v86 mode. */
8619	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8620	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8621	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8622	{
8623	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8624	}
8625
8626	/*
8627	* Load the guest-state into the VMCS.
8628	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8629	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8630	*/
8631	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8632	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8633
8634	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8635	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8636	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8637
8638	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8639	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8640	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8641
8642	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8643	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8644
8645	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8646	if (rcStrict == VINF_SUCCESS)
8647	{ /* likely */ }
8648	else
8649	{
8650	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8651	return rcStrict;
8652	}
8653
8654	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8655	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8656	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8657
8658	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8659	determine we don't have to swap EFER after all. */
8660	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8661	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8662
8663	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8664	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8665
8666	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8667	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8668
8669	/*
8670	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8671	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8672	*/
8673	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8674	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8675
8676	/* Clear any unused and reserved bits. */
8677	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8678
8679	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8680	return rc;
8681	}
8682
8683
8684	/**
8685	* Loads the state shared between the host and guest into the VMCS.
8686	*
8687	* @param pVM The cross context VM structure.
8688	* @param pVCpu The cross context virtual CPU structure.
8689	* @param pCtx Pointer to the guest-CPU context.
8690	*
8691	* @remarks No-long-jump zone!!!
8692	*/
8693	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8694	{
8695	NOREF(pVM);
8696
8697	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8698	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8699
8700	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8701	{
8702	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8703	AssertRC(rc);
8704	}
8705
8706	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8707	{
8708	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8709	AssertRC(rc);
8710
8711	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8712	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8713	{
8714	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8715	AssertRC(rc);
8716	}
8717	}
8718
8719	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8720	{
8721	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8722	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8723	}
8724
8725	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8726	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8727	{
8728	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8729	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8730	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8731	AssertRC(rc);
8732	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8733	}
8734
8735	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8736	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8737	}
8738
8739
8740	/**
8741	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8742	*
8743	* @returns Strict VBox status code (i.e. informational status codes too).
8744	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8745	* without unrestricted guest access and the VMMDev is not presently
8746	* mapped (e.g. EFI32).
8747	*
8748	* @param pVM The cross context VM structure.
8749	* @param pVCpu The cross context virtual CPU structure.
8750	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8751	* out-of-sync. Make sure to update the required fields
8752	* before using them.
8753	*
8754	* @remarks No-long-jump zone!!!
8755	*/
8756	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8757	{
8758	HMVMX_ASSERT_PREEMPT_SAFE();
8759	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8760	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8761
8762	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8763	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8764	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8765	#endif
8766
8767	/*
8768	* RIP is what changes the most often and hence if it's the only bit needing to be
8769	* updated, we shall handle it early for performance reasons.
8770	*/
8771	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8772	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8773	{
8774	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8775	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8776	{ /* likely */}
8777	else
8778	{
8779	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8780	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8781	}
8782	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8783	}
8784	else if (HMCPU_CF_VALUE(pVCpu))
8785	{
8786	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8787	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8788	{ /* likely */}
8789	else
8790	{
8791	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8792	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8793	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8794	return rcStrict;
8795	}
8796	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8797	}
8798
8799	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8800	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8801	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8802	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8803	return rcStrict;
8804	}
8805
8806
8807	/**
8808	* Does the preparations before executing guest code in VT-x.
8809	*
8810	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8811	* recompiler/IEM. We must be cautious what we do here regarding committing
8812	* guest-state information into the VMCS assuming we assuredly execute the
8813	* guest in VT-x mode.
8814	*
8815	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8816	* the common-state (TRPM/forceflags), we must undo those changes so that the
8817	* recompiler/IEM can (and should) use them when it resumes guest execution.
8818	* Otherwise such operations must be done when we can no longer exit to ring-3.
8819	*
8820	* @returns Strict VBox status code (i.e. informational status codes too).
8821	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8822	* have been disabled.
8823	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8824	* double-fault into the guest.
8825	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8826	* dispatched directly.
8827	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8828	*
8829	* @param pVM The cross context VM structure.
8830	* @param pVCpu The cross context virtual CPU structure.
8831	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8832	* out-of-sync. Make sure to update the required fields
8833	* before using them.
8834	* @param pVmxTransient Pointer to the VMX transient structure.
8835	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8836	* us ignore some of the reasons for returning to
8837	* ring-3, and return VINF_EM_DBG_STEPPED if event
8838	* dispatching took place.
8839	*/
8840	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8841	{
8842	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8843
8844	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8845	PGMRZDynMapFlushAutoSet(pVCpu);
8846	#endif
8847
8848	/* Check force flag actions that might require us to go back to ring-3. */
8849	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8850	if (rcStrict == VINF_SUCCESS)
8851	{ /* FFs doesn't get set all the time. */ }
8852	else
8853	return rcStrict;
8854
8855	#ifndef IEM_VERIFICATION_MODE_FULL
8856	/*
8857	* Setup the virtualized-APIC accesses.
8858	*
8859	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8860	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8861	*
8862	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8863	*/
8864	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8865	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8866	&& PDMHasApic(pVM))
8867	{
8868	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8869	Assert(u64MsrApicBase);
8870	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8871
8872	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8873
8874	/* Unalias any existing mapping. */
8875	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8876	AssertRCReturn(rc, rc);
8877
8878	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8879	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8880	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8881	AssertRCReturn(rc, rc);
8882
8883	/* Update the per-VCPU cache of the APIC base MSR. */
8884	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8885	}
8886	#endif /* !IEM_VERIFICATION_MODE_FULL */
8887
8888	if (TRPMHasTrap(pVCpu))
8889	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8890	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8891
8892	/*
8893	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8894	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8895	*/
8896	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8897	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8898	{ /* likely */ }
8899	else
8900	{
8901	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8902	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8903	return rcStrict;
8904	}
8905
8906	/*
8907	* No longjmps to ring-3 from this point on!!!
8908	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8909	* This also disables flushing of the R0-logger instance (if any).
8910	*/
8911	VMMRZCallRing3Disable(pVCpu);
8912
8913	/*
8914	* Load the guest state bits.
8915	*
8916	* We cannot perform longjmps while loading the guest state because we do not preserve the
8917	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8918	* CPU migration.
8919	*
8920	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8921	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8922	* Hence, loading of the guest state needs to be done -after- injection of events.
8923	*/
8924	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8925	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8926	{ /* likely */ }
8927	else
8928	{
8929	VMMRZCallRing3Enable(pVCpu);
8930	return rcStrict;
8931	}
8932
8933	/*
8934	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8935	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8936	*
8937	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8938	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8939	*
8940	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8941	* executing guest code.
8942	*/
8943	pVmxTransient->fEFlags = ASMIntDisableFlags();
8944
8945	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8946	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8947	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8948	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8949	{
8950	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8951	{
8952	pVCpu->hm.s.Event.fPending = false;
8953
8954	/*
8955	* We've injected any pending events. This is really the point of no return (to ring-3).
8956	*
8957	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
8958	* returns from this function, so don't enable them here.
8959	*/
8960	return VINF_SUCCESS;
8961	}
8962
8963	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8964	rcStrict = VINF_EM_RAW_INTERRUPT;
8965	}
8966	else
8967	{
8968	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8969	rcStrict = VINF_EM_RAW_TO_R3;
8970	}
8971
8972	ASMSetFlags(pVmxTransient->fEFlags);
8973	VMMRZCallRing3Enable(pVCpu);
8974
8975	return rcStrict;
8976	}
8977
8978
8979	/**
8980	* Prepares to run guest code in VT-x and we've committed to doing so. This
8981	* means there is no backing out to ring-3 or anywhere else at this
8982	* point.
8983	*
8984	* @param pVM The cross context VM structure.
8985	* @param pVCpu The cross context virtual CPU structure.
8986	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8987	* out-of-sync. Make sure to update the required fields
8988	* before using them.
8989	* @param pVmxTransient Pointer to the VMX transient structure.
8990	*
8991	* @remarks Called with preemption disabled.
8992	* @remarks No-long-jump zone!!!
8993	*/
8994	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8995	{
8996	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8997	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8998	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8999
9000	/*
9001	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9002	*/
9003	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9004	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9005
9006	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9007	if (!CPUMIsGuestFPUStateActive(pVCpu))
9008	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9009	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9010	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9011	#endif
9012
9013	if ( pVCpu->hm.s.fPreloadGuestFpu
9014	&& !CPUMIsGuestFPUStateActive(pVCpu))
9015	{
9016	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9017	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9018	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9019	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9020	}
9021
9022	/*
9023	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9024	*/
9025	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9026	&& pVCpu->hm.s.vmx.cMsrs > 0)
9027	{
9028	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9029	}
9030
9031	/*
9032	* Load the host state bits as we may've been preempted (only happens when
9033	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9034	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9035	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9036	* See @bugref{8432}.
9037	*/
9038	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9039	{
9040	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9041	AssertRC(rc);
9042	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9043	}
9044	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9045
9046	/*
9047	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9048	*/
9049	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9050	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9051	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9052
9053	/* Store status of the shared guest-host state at the time of VM-entry. */
9054	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9055	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9056	{
9057	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9058	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9059	}
9060	else
9061	#endif
9062	{
9063	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9064	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9065	}
9066	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9067
9068	/*
9069	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9070	*/
9071	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9072	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9073
9074	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9075	RTCPUID idCurrentCpu = pCpu->idCpu;
9076	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9077	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9078	{
9079	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9080	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9081	}
9082
9083	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9084	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9085	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9086	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9087
9088	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9089
9090	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9091	to start executing. */
9092
9093	/*
9094	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9095	*/
9096	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9097	{
9098	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9099	{
9100	bool fMsrUpdated;
9101	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9102	AssertRC(rc2);
9103	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9104
9105	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9106	&fMsrUpdated);
9107	AssertRC(rc2);
9108	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9109
9110	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9111	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9112	}
9113	else
9114	{
9115	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9116	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9117	}
9118	}
9119
9120	#ifdef VBOX_STRICT
9121	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9122	hmR0VmxCheckHostEferMsr(pVCpu);
9123	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9124	#endif
9125	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9126	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9127	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9128	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9129	#endif
9130	}
9131
9132
9133	/**
9134	* Performs some essential restoration of state after running guest code in
9135	* VT-x.
9136	*
9137	* @param pVM The cross context VM structure.
9138	* @param pVCpu The cross context virtual CPU structure.
9139	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9140	* out-of-sync. Make sure to update the required fields
9141	* before using them.
9142	* @param pVmxTransient Pointer to the VMX transient structure.
9143	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9144	*
9145	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9146	*
9147	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9148	* unconditionally when it is safe to do so.
9149	*/
9150	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9151	{
9152	NOREF(pVM);
9153
9154	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9155
9156	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9157	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9158	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9159	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9160	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9161	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9162
9163	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9164	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9165
9166	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9167	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9168	Assert(!ASMIntAreEnabled());
9169	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9170
9171	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9172	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9173	{
9174	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9175	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9176	}
9177	#endif
9178
9179	#if HC_ARCH_BITS == 64
9180	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9181	#endif
9182	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9183	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9184	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9185	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9186	#else
9187	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9188	#endif
9189	#ifdef VBOX_STRICT
9190	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9191	#endif
9192	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9193	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
9194
9195	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9196	uint32_t uExitReason;
9197	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9198	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9199	AssertRC(rc);
9200	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9201	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9202
9203	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
9204	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
9205	{
9206	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9207	pVmxTransient->fVMEntryFailed));
9208	return;
9209	}
9210
9211	/*
9212	* Update the VM-exit history array here even if the VM-entry failed due to:
9213	* - Invalid guest state.
9214	* - MSR loading.
9215	* - Machine-check event.
9216	*
9217	* In any of the above cases we will still have a "valid" VM-exit reason
9218	* despite @a fVMEntryFailed being false.
9219	*
9220	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9221	*/
9222	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9223
9224	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
9225	{
9226	/** @todo We can optimize this by only syncing with our force-flags when
9227	* really needed and keeping the VMCS state as it is for most
9228	* VM-exits. */
9229	/* Update the guest interruptibility-state from the VMCS. */
9230	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9231
9232	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9233	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9234	AssertRC(rc);
9235	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9236	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9237	AssertRC(rc);
9238	#endif
9239
9240	/*
9241	* Sync the TPR shadow with our APIC state.
9242	*/
9243	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9244	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9245	{
9246	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9247	AssertRC(rc);
9248	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9249	}
9250	}
9251	}
9252
9253
9254	/**
9255	* Runs the guest code using VT-x the normal way.
9256	*
9257	* @returns VBox status code.
9258	* @param pVM The cross context VM structure.
9259	* @param pVCpu The cross context virtual CPU structure.
9260	* @param pCtx Pointer to the guest-CPU context.
9261	*
9262	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9263	*/
9264	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9265	{
9266	VMXTRANSIENT VmxTransient;
9267	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9268	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9269	uint32_t cLoops = 0;
9270
9271	for (;; cLoops++)
9272	{
9273	Assert(!HMR0SuspendPending());
9274	HMVMX_ASSERT_CPU_SAFE();
9275
9276	/* Preparatory work for running guest code, this may force us to return
9277	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9278	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9279	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9280	if (rcStrict != VINF_SUCCESS)
9281	break;
9282
9283	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9284	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9285	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9286
9287	/* Restore any residual host-state and save any bits shared between host
9288	and guest into the guest-CPU state. Re-enables interrupts! */
9289	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9290
9291	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9292	if (RT_SUCCESS(rcRun))
9293	{ /* very likely */ }
9294	else
9295	{
9296	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9297	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9298	return rcRun;
9299	}
9300
9301	/* Profile the VM-exit. */
9302	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9303	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9304	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9305	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9306	HMVMX_START_EXIT_DISPATCH_PROF();
9307
9308	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9309
9310	/* Handle the VM-exit. */
9311	#ifdef HMVMX_USE_FUNCTION_TABLE
9312	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9313	#else
9314	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9315	#endif
9316	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9317	if (rcStrict == VINF_SUCCESS)
9318	{
9319	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9320	continue; /* likely */
9321	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9322	rcStrict = VINF_EM_RAW_INTERRUPT;
9323	}
9324	break;
9325	}
9326
9327	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9328	return rcStrict;
9329	}
9330
9331
9332
9333	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9334	* probes.
9335	*
9336	* The following few functions and associated structure contains the bloat
9337	* necessary for providing detailed debug events and dtrace probes as well as
9338	* reliable host side single stepping. This works on the principle of
9339	* "subclassing" the normal execution loop and workers. We replace the loop
9340	* method completely and override selected helpers to add necessary adjustments
9341	* to their core operation.
9342	*
9343	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9344	* any performance for debug and analysis features.
9345	*
9346	* @{
9347	*/
9348
9349	/**
9350	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9351	* the debug run loop.
9352	*/
9353	typedef struct VMXRUNDBGSTATE
9354	{
9355	/** The RIP we started executing at. This is for detecting that we stepped. */
9356	uint64_t uRipStart;
9357	/** The CS we started executing with. */
9358	uint16_t uCsStart;
9359
9360	/** Whether we've actually modified the 1st execution control field. */
9361	bool fModifiedProcCtls : 1;
9362	/** Whether we've actually modified the 2nd execution control field. */
9363	bool fModifiedProcCtls2 : 1;
9364	/** Whether we've actually modified the exception bitmap. */
9365	bool fModifiedXcptBitmap : 1;
9366
9367	/** We desire the modified the CR0 mask to be cleared. */
9368	bool fClearCr0Mask : 1;
9369	/** We desire the modified the CR4 mask to be cleared. */
9370	bool fClearCr4Mask : 1;
9371	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9372	uint32_t fCpe1Extra;
9373	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9374	uint32_t fCpe1Unwanted;
9375	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9376	uint32_t fCpe2Extra;
9377	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9378	uint32_t bmXcptExtra;
9379	/** The sequence number of the Dtrace provider settings the state was
9380	* configured against. */
9381	uint32_t uDtraceSettingsSeqNo;
9382	/** VM-exits to check (one bit per VM-exit). */
9383	uint32_t bmExitsToCheck[3];
9384
9385	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9386	uint32_t fProcCtlsInitial;
9387	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9388	uint32_t fProcCtls2Initial;
9389	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9390	uint32_t bmXcptInitial;
9391	} VMXRUNDBGSTATE;
9392	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9393	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9394
9395
9396	/**
9397	* Initializes the VMXRUNDBGSTATE structure.
9398	*
9399	* @param pVCpu The cross context virtual CPU structure of the
9400	* calling EMT.
9401	* @param pCtx The CPU register context to go with @a pVCpu.
9402	* @param pDbgState The structure to initialize.
9403	*/
9404	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9405	{
9406	pDbgState->uRipStart = pCtx->rip;
9407	pDbgState->uCsStart = pCtx->cs.Sel;
9408
9409	pDbgState->fModifiedProcCtls = false;
9410	pDbgState->fModifiedProcCtls2 = false;
9411	pDbgState->fModifiedXcptBitmap = false;
9412	pDbgState->fClearCr0Mask = false;
9413	pDbgState->fClearCr4Mask = false;
9414	pDbgState->fCpe1Extra = 0;
9415	pDbgState->fCpe1Unwanted = 0;
9416	pDbgState->fCpe2Extra = 0;
9417	pDbgState->bmXcptExtra = 0;
9418	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9419	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9420	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9421	}
9422
9423
9424	/**
9425	* Updates the VMSC fields with changes requested by @a pDbgState.
9426	*
9427	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9428	* immediately before executing guest code, i.e. when interrupts are disabled.
9429	* We don't check status codes here as we cannot easily assert or return in the
9430	* latter case.
9431	*
9432	* @param pVCpu The cross context virtual CPU structure.
9433	* @param pDbgState The debug state.
9434	*/
9435	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9436	{
9437	/*
9438	* Ensure desired flags in VMCS control fields are set.
9439	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9440	*
9441	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9442	* there should be no stale data in pCtx at this point.
9443	*/
9444	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9445	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9446	{
9447	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9448	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9449	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9450	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9451	pDbgState->fModifiedProcCtls = true;
9452	}
9453
9454	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9455	{
9456	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9457	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9458	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9459	pDbgState->fModifiedProcCtls2 = true;
9460	}
9461
9462	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9463	{
9464	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9465	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9466	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9467	pDbgState->fModifiedXcptBitmap = true;
9468	}
9469
9470	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9471	{
9472	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9473	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9474	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9475	}
9476
9477	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9478	{
9479	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9480	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9481	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9482	}
9483	}
9484
9485
9486	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9487	{
9488	/*
9489	* Restore VM-exit control settings as we may not reenter this function the
9490	* next time around.
9491	*/
9492	/* We reload the initial value, trigger what we can of recalculations the
9493	next time around. From the looks of things, that's all that's required atm. */
9494	if (pDbgState->fModifiedProcCtls)
9495	{
9496	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9497	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9498	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9499	AssertRCReturn(rc2, rc2);
9500	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9501	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_DEBUG);
9502	}
9503
9504	/* We're currently the only ones messing with this one, so just restore the
9505	cached value and reload the field. */
9506	if ( pDbgState->fModifiedProcCtls2
9507	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9508	{
9509	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9510	AssertRCReturn(rc2, rc2);
9511	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9512	}
9513
9514	/* If we've modified the exception bitmap, we restore it and trigger
9515	reloading and partial recalculation the next time around. */
9516	if (pDbgState->fModifiedXcptBitmap)
9517	{
9518	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9519	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS \| HM_CHANGED_GUEST_CR0);
9520	}
9521
9522	/* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9523	if (pDbgState->fClearCr0Mask)
9524	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9525
9526	/* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9527	if (pDbgState->fClearCr4Mask)
9528	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9529
9530	return rcStrict;
9531	}
9532
9533
9534	/**
9535	* Configures VM-exit controls for current DBGF and DTrace settings.
9536	*
9537	* This updates @a pDbgState and the VMCS execution control fields to reflect
9538	* the necessary VM-exits demanded by DBGF and DTrace.
9539	*
9540	* @param pVM The cross context VM structure.
9541	* @param pVCpu The cross context virtual CPU structure.
9542	* @param pCtx Pointer to the guest-CPU context.
9543	* @param pDbgState The debug state.
9544	* @param pVmxTransient Pointer to the VMX transient structure. May update
9545	* fUpdateTscOffsettingAndPreemptTimer.
9546	*/
9547	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9548	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9549	{
9550	/*
9551	* Take down the dtrace serial number so we can spot changes.
9552	*/
9553	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9554	ASMCompilerBarrier();
9555
9556	/*
9557	* We'll rebuild most of the middle block of data members (holding the
9558	* current settings) as we go along here, so start by clearing it all.
9559	*/
9560	pDbgState->bmXcptExtra = 0;
9561	pDbgState->fCpe1Extra = 0;
9562	pDbgState->fCpe1Unwanted = 0;
9563	pDbgState->fCpe2Extra = 0;
9564	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9565	pDbgState->bmExitsToCheck[i] = 0;
9566
9567	/*
9568	* Software interrupts (INT XXh) - no idea how to trigger these...
9569	*/
9570	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9571	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9572	{
9573	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9574	}
9575
9576	/*
9577	* INT3 breakpoints - triggered by #BP exceptions.
9578	*/
9579	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9580	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9581
9582	/*
9583	* Exception bitmap and XCPT events+probes.
9584	*/
9585	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9586	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9587	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9588
9589	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9590	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9591	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9592	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9593	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9594	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9595	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9596	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9597	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9598	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9599	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9600	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9601	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9602	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9603	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9604	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9605	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9606	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9607
9608	if (pDbgState->bmXcptExtra)
9609	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9610
9611	/*
9612	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9613	*
9614	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9615	* So, when adding/changing/removing please don't forget to update it.
9616	*
9617	* Some of the macros are picking up local variables to save horizontal space,
9618	* (being able to see it in a table is the lesser evil here).
9619	*/
9620	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9621	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9622	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9623	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9624	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9625	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9626	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9627	} else do { } while (0)
9628	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9629	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9630	{ \
9631	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9632	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9633	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9634	} else do { } while (0)
9635	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9636	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9637	{ \
9638	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9639	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9640	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9641	} else do { } while (0)
9642	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9643	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9644	{ \
9645	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9646	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9647	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9648	} else do { } while (0)
9649
9650	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9651	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9652	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9653	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9654	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9655
9656	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9657	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9658	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9659	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9660	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9661	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9662	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9663	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9664	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9665	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9666	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9667	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9668	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9669	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9670	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9671	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9672	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9673	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9674	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9675	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9676	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9677	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9678	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9679	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9680	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9681	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9682	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9683	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9684	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9685	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9686	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9687	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9688	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9689	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9690	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9691	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9692
9693	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9694	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9695	{
9696	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9697	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9698	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9699	AssertRC(rc2);
9700
9701	#if 0 /** @todo fix me */
9702	pDbgState->fClearCr0Mask = true;
9703	pDbgState->fClearCr4Mask = true;
9704	#endif
9705	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9706	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9707	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9708	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9709	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9710	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9711	require clearing here and in the loop if we start using it. */
9712	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9713	}
9714	else
9715	{
9716	if (pDbgState->fClearCr0Mask)
9717	{
9718	pDbgState->fClearCr0Mask = false;
9719	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9720	}
9721	if (pDbgState->fClearCr4Mask)
9722	{
9723	pDbgState->fClearCr4Mask = false;
9724	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9725	}
9726	}
9727	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9728	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9729
9730	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9731	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9732	{
9733	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9734	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9735	}
9736	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9737	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9738
9739	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9740	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9741	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9742	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9743	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9744	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9745	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9746	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9747	#if 0 /** @todo too slow, fix handler. */
9748	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9749	#endif
9750	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9751
9752	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9753	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9754	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9755	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9756	{
9757	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9758	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9759	}
9760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9761	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9762	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9763	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9764
9765	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9766	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9767	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9768	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9769	{
9770	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9771	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9772	}
9773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9774	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9775	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9776	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9777
9778	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9779	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9780	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9781	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9782	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9783	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9784	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9785	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9786	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9787	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9788	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9789	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9790	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9791	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9792	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9793	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9794	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9795	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9796	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9797	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9798	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9799	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9800
9801	#undef IS_EITHER_ENABLED
9802	#undef SET_ONLY_XBM_IF_EITHER_EN
9803	#undef SET_CPE1_XBM_IF_EITHER_EN
9804	#undef SET_CPEU_XBM_IF_EITHER_EN
9805	#undef SET_CPE2_XBM_IF_EITHER_EN
9806
9807	/*
9808	* Sanitize the control stuff.
9809	*/
9810	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9811	if (pDbgState->fCpe2Extra)
9812	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9813	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9814	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9815	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9816	{
9817	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9818	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9819	}
9820
9821	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9822	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9823	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9824	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9825	}
9826
9827
9828	/**
9829	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9830	* appropriate.
9831	*
9832	* The caller has checked the VM-exit against the
9833	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9834	* already, so we don't have to do that either.
9835	*
9836	* @returns Strict VBox status code (i.e. informational status codes too).
9837	* @param pVM The cross context VM structure.
9838	* @param pVCpu The cross context virtual CPU structure.
9839	* @param pMixedCtx Pointer to the guest-CPU context.
9840	* @param pVmxTransient Pointer to the VMX-transient structure.
9841	* @param uExitReason The VM-exit reason.
9842	*
9843	* @remarks The name of this function is displayed by dtrace, so keep it short
9844	* and to the point. No longer than 33 chars long, please.
9845	*/
9846	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9847	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9848	{
9849	/*
9850	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9851	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9852	*
9853	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9854	* does. Must add/change/remove both places. Same ordering, please.
9855	*
9856	* Added/removed events must also be reflected in the next section
9857	* where we dispatch dtrace events.
9858	*/
9859	bool fDtrace1 = false;
9860	bool fDtrace2 = false;
9861	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9862	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9863	uint32_t uEventArg = 0;
9864	#define SET_EXIT(a_EventSubName) \
9865	do { \
9866	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9867	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9868	} while (0)
9869	#define SET_BOTH(a_EventSubName) \
9870	do { \
9871	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9872	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9873	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9874	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9875	} while (0)
9876	switch (uExitReason)
9877	{
9878	case VMX_EXIT_MTF:
9879	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9880
9881	case VMX_EXIT_XCPT_OR_NMI:
9882	{
9883	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9884	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9885	{
9886	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9887	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9888	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9889	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9890	{
9891	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9892	{
9893	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9894	uEventArg = pVmxTransient->uExitIntErrorCode;
9895	}
9896	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9897	switch (enmEvent1)
9898	{
9899	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9900	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9901	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9902	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9903	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9904	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9905	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9906	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9907	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9908	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9909	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9910	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9911	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9912	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9913	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9914	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9915	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9916	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9917	default: break;
9918	}
9919	}
9920	else
9921	AssertFailed();
9922	break;
9923
9924	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9925	uEventArg = idxVector;
9926	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9927	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9928	break;
9929	}
9930	break;
9931	}
9932
9933	case VMX_EXIT_TRIPLE_FAULT:
9934	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9935	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9936	break;
9937	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9938	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9939	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9940	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9941	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9942
9943	/* Instruction specific VM-exits: */
9944	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9945	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9946	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9947	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9948	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9949	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9950	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9951	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9952	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9953	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9954	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9955	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9956	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9957	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9958	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9959	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9960	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9961	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9962	case VMX_EXIT_MOV_CRX:
9963	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9964	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9965	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9966	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9967	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9968	SET_BOTH(CRX_READ);
9969	else
9970	SET_BOTH(CRX_WRITE);
9971	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9972	break;
9973	case VMX_EXIT_MOV_DRX:
9974	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9975	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9976	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9977	SET_BOTH(DRX_READ);
9978	else
9979	SET_BOTH(DRX_WRITE);
9980	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9981	break;
9982	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9983	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9984	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9985	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9986	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9987	case VMX_EXIT_XDTR_ACCESS:
9988	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9989	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9990	{
9991	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9992	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9993	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9994	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9995	}
9996	break;
9997
9998	case VMX_EXIT_TR_ACCESS:
9999	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10000	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10001	{
10002	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10003	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10004	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10005	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10006	}
10007	break;
10008
10009	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10010	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10011	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10012	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10013	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10014	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10015	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10016	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10017	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10018	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10019	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10020
10021	/* Events that aren't relevant at this point. */
10022	case VMX_EXIT_EXT_INT:
10023	case VMX_EXIT_INT_WINDOW:
10024	case VMX_EXIT_NMI_WINDOW:
10025	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10026	case VMX_EXIT_PREEMPT_TIMER:
10027	case VMX_EXIT_IO_INSTR:
10028	break;
10029
10030	/* Errors and unexpected events. */
10031	case VMX_EXIT_INIT_SIGNAL:
10032	case VMX_EXIT_SIPI:
10033	case VMX_EXIT_IO_SMI:
10034	case VMX_EXIT_SMI:
10035	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10036	case VMX_EXIT_ERR_MSR_LOAD:
10037	case VMX_EXIT_ERR_MACHINE_CHECK:
10038	break;
10039
10040	default:
10041	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10042	break;
10043	}
10044	#undef SET_BOTH
10045	#undef SET_EXIT
10046
10047	/*
10048	* Dtrace tracepoints go first. We do them here at once so we don't
10049	* have to copy the guest state saving and stuff a few dozen times.
10050	* Down side is that we've got to repeat the switch, though this time
10051	* we use enmEvent since the probes are a subset of what DBGF does.
10052	*/
10053	if (fDtrace1 \|\| fDtrace2)
10054	{
10055	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10056	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10057	switch (enmEvent1)
10058	{
10059	/** @todo consider which extra parameters would be helpful for each probe. */
10060	case DBGFEVENT_END: break;
10061	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10062	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10063	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10064	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10065	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10066	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10067	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10068	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10069	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10070	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10071	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10072	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10073	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10074	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10075	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10076	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10077	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10078	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10079	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10080	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10081	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10082	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10083	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10084	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10085	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10086	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10087	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10088	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10089	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10090	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10091	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10092	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10093	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10094	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10095	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10096	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10097	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10098	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10099	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10100	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10101	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10102	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10103	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10104	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10105	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10106	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10107	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10108	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10109	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10110	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10111	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10112	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10113	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10114	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10115	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10116	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10117	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10118	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10119	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10120	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10121	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10122	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10123	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10124	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10125	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10126	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10127	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10128	}
10129	switch (enmEvent2)
10130	{
10131	/** @todo consider which extra parameters would be helpful for each probe. */
10132	case DBGFEVENT_END: break;
10133	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10134	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10135	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10136	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10137	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10138	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10139	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10140	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10141	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10142	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10143	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10144	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10145	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10146	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10147	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10148	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10149	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10150	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10151	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10152	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10153	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10154	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10155	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10156	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10157	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10158	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10159	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10160	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10161	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10162	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10163	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10164	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10165	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10166	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10167	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10173	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10174	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10175	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10176	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10177	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10178	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10179	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10180	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10181	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10182	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10183	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10184	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10185	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10186	}
10187	}
10188
10189	/*
10190	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10191	* the DBGF call will do a full check).
10192	*
10193	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10194	* Note! If we have to events, we prioritize the first, i.e. the instruction
10195	* one, in order to avoid event nesting.
10196	*/
10197	if ( enmEvent1 != DBGFEVENT_END
10198	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10199	{
10200	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10201	if (rcStrict != VINF_SUCCESS)
10202	return rcStrict;
10203	}
10204	else if ( enmEvent2 != DBGFEVENT_END
10205	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10206	{
10207	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10208	if (rcStrict != VINF_SUCCESS)
10209	return rcStrict;
10210	}
10211
10212	return VINF_SUCCESS;
10213	}
10214
10215
10216	/**
10217	* Single-stepping VM-exit filtering.
10218	*
10219	* This is preprocessing the VM-exits and deciding whether we've gotten far
10220	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10221	* handling is performed.
10222	*
10223	* @returns Strict VBox status code (i.e. informational status codes too).
10224	* @param pVM The cross context VM structure.
10225	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10226	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10227	* out-of-sync. Make sure to update the required
10228	* fields before using them.
10229	* @param pVmxTransient Pointer to the VMX-transient structure.
10230	* @param uExitReason The VM-exit reason.
10231	* @param pDbgState The debug state.
10232	*/
10233	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10234	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10235	{
10236	/*
10237	* Expensive (saves context) generic dtrace VM-exit probe.
10238	*/
10239	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10240	{ /* more likely */ }
10241	else
10242	{
10243	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10244	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10245	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10246	}
10247
10248	/*
10249	* Check for host NMI, just to get that out of the way.
10250	*/
10251	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10252	{ /* normally likely */ }
10253	else
10254	{
10255	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10256	AssertRCReturn(rc2, rc2);
10257	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10258	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10259	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10260	}
10261
10262	/*
10263	* Check for single stepping event if we're stepping.
10264	*/
10265	if (pVCpu->hm.s.fSingleInstruction)
10266	{
10267	switch (uExitReason)
10268	{
10269	case VMX_EXIT_MTF:
10270	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10271
10272	/* Various events: */
10273	case VMX_EXIT_XCPT_OR_NMI:
10274	case VMX_EXIT_EXT_INT:
10275	case VMX_EXIT_TRIPLE_FAULT:
10276	case VMX_EXIT_INT_WINDOW:
10277	case VMX_EXIT_NMI_WINDOW:
10278	case VMX_EXIT_TASK_SWITCH:
10279	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10280	case VMX_EXIT_APIC_ACCESS:
10281	case VMX_EXIT_EPT_VIOLATION:
10282	case VMX_EXIT_EPT_MISCONFIG:
10283	case VMX_EXIT_PREEMPT_TIMER:
10284
10285	/* Instruction specific VM-exits: */
10286	case VMX_EXIT_CPUID:
10287	case VMX_EXIT_GETSEC:
10288	case VMX_EXIT_HLT:
10289	case VMX_EXIT_INVD:
10290	case VMX_EXIT_INVLPG:
10291	case VMX_EXIT_RDPMC:
10292	case VMX_EXIT_RDTSC:
10293	case VMX_EXIT_RSM:
10294	case VMX_EXIT_VMCALL:
10295	case VMX_EXIT_VMCLEAR:
10296	case VMX_EXIT_VMLAUNCH:
10297	case VMX_EXIT_VMPTRLD:
10298	case VMX_EXIT_VMPTRST:
10299	case VMX_EXIT_VMREAD:
10300	case VMX_EXIT_VMRESUME:
10301	case VMX_EXIT_VMWRITE:
10302	case VMX_EXIT_VMXOFF:
10303	case VMX_EXIT_VMXON:
10304	case VMX_EXIT_MOV_CRX:
10305	case VMX_EXIT_MOV_DRX:
10306	case VMX_EXIT_IO_INSTR:
10307	case VMX_EXIT_RDMSR:
10308	case VMX_EXIT_WRMSR:
10309	case VMX_EXIT_MWAIT:
10310	case VMX_EXIT_MONITOR:
10311	case VMX_EXIT_PAUSE:
10312	case VMX_EXIT_XDTR_ACCESS:
10313	case VMX_EXIT_TR_ACCESS:
10314	case VMX_EXIT_INVEPT:
10315	case VMX_EXIT_RDTSCP:
10316	case VMX_EXIT_INVVPID:
10317	case VMX_EXIT_WBINVD:
10318	case VMX_EXIT_XSETBV:
10319	case VMX_EXIT_RDRAND:
10320	case VMX_EXIT_INVPCID:
10321	case VMX_EXIT_VMFUNC:
10322	case VMX_EXIT_RDSEED:
10323	case VMX_EXIT_XSAVES:
10324	case VMX_EXIT_XRSTORS:
10325	{
10326	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10327	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10328	AssertRCReturn(rc2, rc2);
10329	if ( pMixedCtx->rip != pDbgState->uRipStart
10330	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10331	return VINF_EM_DBG_STEPPED;
10332	break;
10333	}
10334
10335	/* Errors and unexpected events: */
10336	case VMX_EXIT_INIT_SIGNAL:
10337	case VMX_EXIT_SIPI:
10338	case VMX_EXIT_IO_SMI:
10339	case VMX_EXIT_SMI:
10340	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10341	case VMX_EXIT_ERR_MSR_LOAD:
10342	case VMX_EXIT_ERR_MACHINE_CHECK:
10343	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10344	break;
10345
10346	default:
10347	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10348	break;
10349	}
10350	}
10351
10352	/*
10353	* Check for debugger event breakpoints and dtrace probes.
10354	*/
10355	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10356	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10357	{
10358	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10359	if (rcStrict != VINF_SUCCESS)
10360	return rcStrict;
10361	}
10362
10363	/*
10364	* Normal processing.
10365	*/
10366	#ifdef HMVMX_USE_FUNCTION_TABLE
10367	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10368	#else
10369	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10370	#endif
10371	}
10372
10373
10374	/**
10375	* Single steps guest code using VT-x.
10376	*
10377	* @returns Strict VBox status code (i.e. informational status codes too).
10378	* @param pVM The cross context VM structure.
10379	* @param pVCpu The cross context virtual CPU structure.
10380	* @param pCtx Pointer to the guest-CPU context.
10381	*
10382	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10383	*/
10384	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10385	{
10386	VMXTRANSIENT VmxTransient;
10387	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10388
10389	/* Set HMCPU indicators. */
10390	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10391	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10392	pVCpu->hm.s.fDebugWantRdTscExit = false;
10393	pVCpu->hm.s.fUsingDebugLoop = true;
10394
10395	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10396	VMXRUNDBGSTATE DbgState;
10397	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10398	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10399
10400	/*
10401	* The loop.
10402	*/
10403	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10404	for (uint32_t cLoops = 0; ; cLoops++)
10405	{
10406	Assert(!HMR0SuspendPending());
10407	HMVMX_ASSERT_CPU_SAFE();
10408	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10409
10410	/*
10411	* Preparatory work for running guest code, this may force us to return
10412	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10413	*/
10414	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10415	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10416	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10417	if (rcStrict != VINF_SUCCESS)
10418	break;
10419
10420	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10421	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10422
10423	/*
10424	* Now we can run the guest code.
10425	*/
10426	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10427
10428	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10429
10430	/*
10431	* Restore any residual host-state and save any bits shared between host
10432	* and guest into the guest-CPU state. Re-enables interrupts!
10433	*/
10434	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10435
10436	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10437	if (RT_SUCCESS(rcRun))
10438	{ /* very likely */ }
10439	else
10440	{
10441	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10442	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10443	return rcRun;
10444	}
10445
10446	/* Profile the VM-exit. */
10447	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10448	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10449	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10450	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10451	HMVMX_START_EXIT_DISPATCH_PROF();
10452
10453	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10454
10455	/*
10456	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10457	*/
10458	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10459	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10460	if (rcStrict != VINF_SUCCESS)
10461	break;
10462	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10463	{
10464	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10465	rcStrict = VINF_EM_RAW_INTERRUPT;
10466	break;
10467	}
10468
10469	/*
10470	* Stepping: Did the RIP change, if so, consider it a single step.
10471	* Otherwise, make sure one of the TFs gets set.
10472	*/
10473	if (fStepping)
10474	{
10475	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10476	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10477	AssertRCReturn(rc2, rc2);
10478	if ( pCtx->rip != DbgState.uRipStart
10479	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10480	{
10481	rcStrict = VINF_EM_DBG_STEPPED;
10482	break;
10483	}
10484	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10485	}
10486
10487	/*
10488	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10489	*/
10490	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10491	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10492	}
10493
10494	/*
10495	* Clear the X86_EFL_TF if necessary.
10496	*/
10497	if (pVCpu->hm.s.fClearTrapFlag)
10498	{
10499	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10500	AssertRCReturn(rc2, rc2);
10501	pVCpu->hm.s.fClearTrapFlag = false;
10502	pCtx->eflags.Bits.u1TF = 0;
10503	}
10504	/** @todo there seems to be issues with the resume flag when the monitor trap
10505	* flag is pending without being used. Seen early in bios init when
10506	* accessing APIC page in protected mode. */
10507
10508	/*
10509	* Restore VM-exit control settings as we may not reenter this function the
10510	* next time around.
10511	*/
10512	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10513
10514	/* Restore HMCPU indicators. */
10515	pVCpu->hm.s.fUsingDebugLoop = false;
10516	pVCpu->hm.s.fDebugWantRdTscExit = false;
10517	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10518
10519	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10520	return rcStrict;
10521	}
10522
10523
10524	/** @} */
10525
10526
10527	/**
10528	* Checks if any expensive dtrace probes are enabled and we should go to the
10529	* debug loop.
10530	*
10531	* @returns true if we should use debug loop, false if not.
10532	*/
10533	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10534	{
10535	/* It's probably faster to OR the raw 32-bit counter variables together.
10536	Since the variables are in an array and the probes are next to one
10537	another (more or less), we have good locality. So, better read
10538	eight-nine cache lines ever time and only have one conditional, than
10539	128+ conditionals, right? */
10540	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10541	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10542	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10543	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10544	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10545	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10546	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10547	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10548	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10549	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10550	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10551	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10552	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10553	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10554	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10555	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10556	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10557	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10558	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10559	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10560	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10561	) != 0
10562	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10563	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10564	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10565	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10566	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10567	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10568	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10569	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10570	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10571	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10572	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10573	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10574	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10575	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10576	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10577	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10578	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10579	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10580	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10581	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10582	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10583	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10584	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10585	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10586	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10587	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10588	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10589	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10590	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10591	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10592	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10593	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10594	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10595	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10596	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10597	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10598	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10599	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10600	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10601	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10602	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10603	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10604	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10605	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10606	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10607	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10608	) != 0
10609	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10610	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10611	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10612	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10613	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10614	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10615	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10616	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10617	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10618	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10619	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10620	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10621	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10622	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10623	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10624	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10625	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10626	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10627	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10628	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10629	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10630	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10631	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10632	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10633	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10634	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10635	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10636	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10637	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10638	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10639	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10640	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10641	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10642	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10643	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10644	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10645	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10646	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10647	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10648	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10649	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10650	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10651	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10652	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10653	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10654	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10655	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10656	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10657	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10658	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10659	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10660	) != 0;
10661	}
10662
10663
10664	/**
10665	* Runs the guest code using VT-x.
10666	*
10667	* @returns Strict VBox status code (i.e. informational status codes too).
10668	* @param pVM The cross context VM structure.
10669	* @param pVCpu The cross context virtual CPU structure.
10670	* @param pCtx Pointer to the guest-CPU context.
10671	*/
10672	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10673	{
10674	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10675	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10676	HMVMX_ASSERT_PREEMPT_SAFE();
10677
10678	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10679
10680	VBOXSTRICTRC rcStrict;
10681	if ( !pVCpu->hm.s.fUseDebugLoop
10682	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10683	&& !DBGFIsStepping(pVCpu)
10684	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10685	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10686	else
10687	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10688
10689	if (rcStrict == VERR_EM_INTERPRETER)
10690	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10691	else if (rcStrict == VINF_EM_RESET)
10692	rcStrict = VINF_EM_TRIPLE_FAULT;
10693
10694	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10695	if (RT_FAILURE(rc2))
10696	{
10697	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10698	rcStrict = rc2;
10699	}
10700	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10701	return rcStrict;
10702	}
10703
10704
10705	#ifndef HMVMX_USE_FUNCTION_TABLE
10706	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10707	{
10708	# ifdef DEBUG_ramshankar
10709	# define RETURN_EXIT_CALL(a_CallExpr) \
10710	do { \
10711	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10712	VBOXSTRICTRC rcStrict = a_CallExpr; \
10713	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10714	return rcStrict; \
10715	} while (0)
10716	# else
10717	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10718	# endif
10719	switch (rcReason)
10720	{
10721	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10722	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10723	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10724	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10725	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10726	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10727	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10728	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10729	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10730	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10731	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10732	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10733	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10734	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10735	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10736	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10737	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10738	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10739	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10740	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10741	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10742	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10743	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10744	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10745	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10746	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10747	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10748	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10749	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10750	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10751	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10752	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10753	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10754	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10755
10756	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10757	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10758	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10759	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10760	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10761	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10762	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10763	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10764	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10765
10766	case VMX_EXIT_VMCLEAR:
10767	case VMX_EXIT_VMLAUNCH:
10768	case VMX_EXIT_VMPTRLD:
10769	case VMX_EXIT_VMPTRST:
10770	case VMX_EXIT_VMREAD:
10771	case VMX_EXIT_VMRESUME:
10772	case VMX_EXIT_VMWRITE:
10773	case VMX_EXIT_VMXOFF:
10774	case VMX_EXIT_VMXON:
10775	case VMX_EXIT_INVEPT:
10776	case VMX_EXIT_INVVPID:
10777	case VMX_EXIT_VMFUNC:
10778	case VMX_EXIT_XSAVES:
10779	case VMX_EXIT_XRSTORS:
10780	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10781	case VMX_EXIT_ENCLS:
10782	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10783	case VMX_EXIT_PML_FULL:
10784	default:
10785	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10786	}
10787	#undef RETURN_EXIT_CALL
10788	}
10789	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10790
10791
10792	#ifdef VBOX_STRICT
10793	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10794	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10795	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10796
10797	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10798	do { \
10799	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10800	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10801	} while (0)
10802
10803	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10804	do { \
10805	AssertPtr(pVCpu); \
10806	AssertPtr(pMixedCtx); \
10807	AssertPtr(pVmxTransient); \
10808	Assert(pVmxTransient->fVMEntryFailed == false); \
10809	Assert(ASMIntAreEnabled()); \
10810	HMVMX_ASSERT_PREEMPT_SAFE(); \
10811	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10812	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10813	HMVMX_ASSERT_PREEMPT_SAFE(); \
10814	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10815	HMVMX_ASSERT_PREEMPT_CPUID(); \
10816	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10817	} while (0)
10818
10819	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10820	do { \
10821	Log4Func(("\n")); \
10822	} while (0)
10823	#else /* nonstrict builds: */
10824	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10825	do { \
10826	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10827	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10828	} while (0)
10829	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10830	#endif
10831
10832
10833	/**
10834	* Advances the guest RIP by the specified number of bytes.
10835	*
10836	* @param pVCpu The cross context virtual CPU structure.
10837	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10838	* out-of-sync. Make sure to update the required fields
10839	* before using them.
10840	* @param cbInstr Number of bytes to advance the RIP by.
10841	*
10842	* @remarks No-long-jump zone!!!
10843	*/
10844	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10845	{
10846	/* Advance the RIP. */
10847	pMixedCtx->rip += cbInstr;
10848	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10849
10850	/* Update interrupt inhibition. */
10851	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10852	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10853	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10854	}
10855
10856
10857	/**
10858	* Advances the guest RIP after reading it from the VMCS.
10859	*
10860	* @returns VBox status code, no informational status codes.
10861	* @param pVCpu The cross context virtual CPU structure.
10862	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10863	* out-of-sync. Make sure to update the required fields
10864	* before using them.
10865	* @param pVmxTransient Pointer to the VMX transient structure.
10866	*
10867	* @remarks No-long-jump zone!!!
10868	*/
10869	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10870	{
10871	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10872	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10873	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10874	AssertRCReturn(rc, rc);
10875
10876	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10877
10878	/*
10879	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10880	* pending debug exception field as it takes care of priority of events.
10881	*
10882	* See Intel spec. 32.2.1 "Debug Exceptions".
10883	*/
10884	if ( !pVCpu->hm.s.fSingleInstruction
10885	&& pMixedCtx->eflags.Bits.u1TF)
10886	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10887
10888	return VINF_SUCCESS;
10889	}
10890
10891
10892	/**
10893	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10894	* and update error record fields accordingly.
10895	*
10896	* @return VMX_IGS_* return codes.
10897	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10898	* wrong with the guest state.
10899	*
10900	* @param pVM The cross context VM structure.
10901	* @param pVCpu The cross context virtual CPU structure.
10902	* @param pCtx Pointer to the guest-CPU state.
10903	*
10904	* @remarks This function assumes our cache of the VMCS controls
10905	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10906	*/
10907	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10908	{
10909	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10910	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10911	uError = (err); \
10912	break; \
10913	} else do { } while (0)
10914
10915	int rc;
10916	uint32_t uError = VMX_IGS_ERROR;
10917	uint32_t u32Val;
10918	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10919
10920	do
10921	{
10922	/*
10923	* CR0.
10924	*/
10925	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10926	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10927	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10928	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10929	if (fUnrestrictedGuest)
10930	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10931
10932	uint32_t u32GuestCR0;
10933	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10934	AssertRCBreak(rc);
10935	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10936	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10937	if ( !fUnrestrictedGuest
10938	&& (u32GuestCR0 & X86_CR0_PG)
10939	&& !(u32GuestCR0 & X86_CR0_PE))
10940	{
10941	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10942	}
10943
10944	/*
10945	* CR4.
10946	*/
10947	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10948	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10949
10950	uint32_t u32GuestCR4;
10951	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10952	AssertRCBreak(rc);
10953	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10954	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10955
10956	/*
10957	* IA32_DEBUGCTL MSR.
10958	*/
10959	uint64_t u64Val;
10960	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10961	AssertRCBreak(rc);
10962	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10963	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10964	{
10965	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10966	}
10967	uint64_t u64DebugCtlMsr = u64Val;
10968
10969	#ifdef VBOX_STRICT
10970	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10971	AssertRCBreak(rc);
10972	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10973	#endif
10974	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10975
10976	/*
10977	* RIP and RFLAGS.
10978	*/
10979	uint32_t u32Eflags;
10980	#if HC_ARCH_BITS == 64
10981	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10982	AssertRCBreak(rc);
10983	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10984	if ( !fLongModeGuest
10985	\|\| !pCtx->cs.Attr.n.u1Long)
10986	{
10987	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10988	}
10989	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10990	* must be identical if the "IA-32e mode guest" VM-entry
10991	* control is 1 and CS.L is 1. No check applies if the
10992	* CPU supports 64 linear-address bits. */
10993
10994	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10995	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10996	AssertRCBreak(rc);
10997	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10998	VMX_IGS_RFLAGS_RESERVED);
10999	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11000	u32Eflags = u64Val;
11001	#else
11002	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11003	AssertRCBreak(rc);
11004	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11005	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11006	#endif
11007
11008	if ( fLongModeGuest
11009	\|\| ( fUnrestrictedGuest
11010	&& !(u32GuestCR0 & X86_CR0_PE)))
11011	{
11012	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11013	}
11014
11015	uint32_t u32EntryInfo;
11016	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11017	AssertRCBreak(rc);
11018	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11019	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11020	{
11021	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11022	}
11023
11024	/*
11025	* 64-bit checks.
11026	*/
11027	#if HC_ARCH_BITS == 64
11028	if (fLongModeGuest)
11029	{
11030	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11031	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11032	}
11033
11034	if ( !fLongModeGuest
11035	&& (u32GuestCR4 & X86_CR4_PCIDE))
11036	{
11037	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11038	}
11039
11040	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11041	* 51:32 beyond the processor's physical-address width are 0. */
11042
11043	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11044	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11045	{
11046	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11047	}
11048
11049	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11050	AssertRCBreak(rc);
11051	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11052
11053	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11054	AssertRCBreak(rc);
11055	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11056	#endif
11057
11058	/*
11059	* PERF_GLOBAL MSR.
11060	*/
11061	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11062	{
11063	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11064	AssertRCBreak(rc);
11065	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11066	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11067	}
11068
11069	/*
11070	* PAT MSR.
11071	*/
11072	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11073	{
11074	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11075	AssertRCBreak(rc);
11076	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11077	for (unsigned i = 0; i < 8; i++)
11078	{
11079	uint8_t u8Val = (u64Val & 0xff);
11080	if ( u8Val != 0 /* UC */
11081	&& u8Val != 1 /* WC */
11082	&& u8Val != 4 /* WT */
11083	&& u8Val != 5 /* WP */
11084	&& u8Val != 6 /* WB */
11085	&& u8Val != 7 /* UC- */)
11086	{
11087	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11088	}
11089	u64Val >>= 8;
11090	}
11091	}
11092
11093	/*
11094	* EFER MSR.
11095	*/
11096	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11097	{
11098	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11099	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11100	AssertRCBreak(rc);
11101	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11102	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11103	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11104	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11105	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11106	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11107	\|\| !(u32GuestCR0 & X86_CR0_PG)
11108	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11109	VMX_IGS_EFER_LMA_LME_MISMATCH);
11110	}
11111
11112	/*
11113	* Segment registers.
11114	*/
11115	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11116	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11117	if (!(u32Eflags & X86_EFL_VM))
11118	{
11119	/* CS */
11120	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11121	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11122	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11123	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11124	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11125	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11126	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11127	/* CS cannot be loaded with NULL in protected mode. */
11128	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11129	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11130	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11131	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11132	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11133	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11134	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11135	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11136	else
11137	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11138
11139	/* SS */
11140	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11141	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11142	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11143	if ( !(pCtx->cr0 & X86_CR0_PE)
11144	\|\| pCtx->cs.Attr.n.u4Type == 3)
11145	{
11146	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11147	}
11148	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11149	{
11150	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11151	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11152	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11153	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11154	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11155	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11156	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11157	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11158	}
11159
11160	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11161	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11162	{
11163	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11164	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11165	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11166	\|\| pCtx->ds.Attr.n.u4Type > 11
11167	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11168	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11169	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11170	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11171	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11172	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11173	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11174	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11175	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11176	}
11177	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11178	{
11179	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11180	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11181	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11182	\|\| pCtx->es.Attr.n.u4Type > 11
11183	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11184	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11185	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11186	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11187	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11188	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11189	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11190	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11191	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11192	}
11193	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11194	{
11195	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11196	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11197	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11198	\|\| pCtx->fs.Attr.n.u4Type > 11
11199	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11200	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11201	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11202	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11203	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11204	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11205	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11206	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11207	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11208	}
11209	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11210	{
11211	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11212	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11213	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11214	\|\| pCtx->gs.Attr.n.u4Type > 11
11215	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11216	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11217	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11218	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11219	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11220	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11221	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11222	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11223	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11224	}
11225	/* 64-bit capable CPUs. */
11226	#if HC_ARCH_BITS == 64
11227	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11228	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11229	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11230	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11231	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11232	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11233	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11234	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11235	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11236	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11237	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11238	#endif
11239	}
11240	else
11241	{
11242	/* V86 mode checks. */
11243	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11244	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11245	{
11246	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11247	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11248	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11249	}
11250	else
11251	{
11252	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11253	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11254	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11255	}
11256
11257	/* CS */
11258	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11259	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11260	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11261	/* SS */
11262	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11263	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11264	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11265	/* DS */
11266	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11267	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11268	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11269	/* ES */
11270	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11271	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11272	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11273	/* FS */
11274	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11275	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11276	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11277	/* GS */
11278	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11279	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11280	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11281	/* 64-bit capable CPUs. */
11282	#if HC_ARCH_BITS == 64
11283	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11284	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11285	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11286	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11287	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11288	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11289	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11290	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11291	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11292	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11293	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11294	#endif
11295	}
11296
11297	/*
11298	* TR.
11299	*/
11300	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11301	/* 64-bit capable CPUs. */
11302	#if HC_ARCH_BITS == 64
11303	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11304	#endif
11305	if (fLongModeGuest)
11306	{
11307	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11308	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11309	}
11310	else
11311	{
11312	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11313	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11314	VMX_IGS_TR_ATTR_TYPE_INVALID);
11315	}
11316	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11317	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11318	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11319	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11320	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11321	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11322	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11323	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11324
11325	/*
11326	* GDTR and IDTR.
11327	*/
11328	#if HC_ARCH_BITS == 64
11329	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11330	AssertRCBreak(rc);
11331	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11332
11333	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11334	AssertRCBreak(rc);
11335	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11336	#endif
11337
11338	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11339	AssertRCBreak(rc);
11340	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11341
11342	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11343	AssertRCBreak(rc);
11344	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11345
11346	/*
11347	* Guest Non-Register State.
11348	*/
11349	/* Activity State. */
11350	uint32_t u32ActivityState;
11351	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11352	AssertRCBreak(rc);
11353	HMVMX_CHECK_BREAK( !u32ActivityState
11354	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11355	VMX_IGS_ACTIVITY_STATE_INVALID);
11356	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11357	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11358	uint32_t u32IntrState;
11359	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11360	AssertRCBreak(rc);
11361	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11362	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11363	{
11364	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11365	}
11366
11367	/** @todo Activity state and injecting interrupts. Left as a todo since we
11368	* currently don't use activity states but ACTIVE. */
11369
11370	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11371	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11372
11373	/* Guest interruptibility-state. */
11374	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11375	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11376	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11377	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11378	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11379	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11380	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11381	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11382	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11383	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11384	{
11385	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11386	{
11387	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11388	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11389	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11390	}
11391	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11392	{
11393	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11394	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11395	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11396	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11397	}
11398	}
11399	/** @todo Assumes the processor is not in SMM. */
11400	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11401	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11402	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11403	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11404	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11405	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11406	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11407	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11408	{
11409	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11410	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11411	}
11412
11413	/* Pending debug exceptions. */
11414	#if HC_ARCH_BITS == 64
11415	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11416	AssertRCBreak(rc);
11417	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11418	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11419	u32Val = u64Val; /* For pending debug exceptions checks below. */
11420	#else
11421	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11422	AssertRCBreak(rc);
11423	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11424	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11425	#endif
11426
11427	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11428	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11429	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11430	{
11431	if ( (u32Eflags & X86_EFL_TF)
11432	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11433	{
11434	/* Bit 14 is PendingDebug.BS. */
11435	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11436	}
11437	if ( !(u32Eflags & X86_EFL_TF)
11438	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11439	{
11440	/* Bit 14 is PendingDebug.BS. */
11441	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11442	}
11443	}
11444
11445	/* VMCS link pointer. */
11446	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11447	AssertRCBreak(rc);
11448	if (u64Val != UINT64_C(0xffffffffffffffff))
11449	{
11450	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11451	/** @todo Bits beyond the processor's physical-address width MBZ. */
11452	/** @todo 32-bit located in memory referenced by value of this field (as a
11453	* physical address) must contain the processor's VMCS revision ID. */
11454	/** @todo SMM checks. */
11455	}
11456
11457	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11458	* not using Nested Paging? */
11459	if ( pVM->hm.s.fNestedPaging
11460	&& !fLongModeGuest
11461	&& CPUMIsGuestInPAEModeEx(pCtx))
11462	{
11463	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11464	AssertRCBreak(rc);
11465	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11466
11467	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11468	AssertRCBreak(rc);
11469	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11470
11471	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11472	AssertRCBreak(rc);
11473	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11474
11475	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11476	AssertRCBreak(rc);
11477	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11478	}
11479
11480	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11481	if (uError == VMX_IGS_ERROR)
11482	uError = VMX_IGS_REASON_NOT_FOUND;
11483	} while (0);
11484
11485	pVCpu->hm.s.u32HMError = uError;
11486	return uError;
11487
11488	#undef HMVMX_ERROR_BREAK
11489	#undef HMVMX_CHECK_BREAK
11490	}
11491
11492	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11493	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11494	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11495
11496	/** @name VM-exit handlers.
11497	* @{
11498	*/
11499
11500	/**
11501	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11502	*/
11503	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11504	{
11505	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11507	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11508	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11509	return VINF_SUCCESS;
11510	return VINF_EM_RAW_INTERRUPT;
11511	}
11512
11513
11514	/**
11515	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11516	*/
11517	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11518	{
11519	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11520	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11521
11522	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11523	AssertRCReturn(rc, rc);
11524
11525	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11526	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11527	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11528	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11529
11530	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11531	{
11532	/*
11533	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11534	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11535	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11536	*
11537	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11538	*/
11539	VMXDispatchHostNmi();
11540	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11541	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11542	return VINF_SUCCESS;
11543	}
11544
11545	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11546	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11547	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11548	{ /* likely */ }
11549	else
11550	{
11551	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11552	rcStrictRc1 = VINF_SUCCESS;
11553	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11554	return rcStrictRc1;
11555	}
11556
11557	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11558	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11559	switch (uIntType)
11560	{
11561	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11562	Assert(uVector == X86_XCPT_DB);
11563	/* fall thru */
11564	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11565	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11566	/* fall thru */
11567	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11568	{
11569	/*
11570	* If there's any exception caused as a result of event injection, go back to
11571	* the interpreter. The page-fault case is complicated and we manually handle
11572	* any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11573	* handled in hmR0VmxCheckExitDueToEventDelivery.
11574	*/
11575	if (!pVCpu->hm.s.Event.fPending)
11576	{ /* likely */ }
11577	else if ( uVector != X86_XCPT_PF
11578	&& uVector != X86_XCPT_AC)
11579	{
11580	/** @todo Why do we need to fallback to the interpreter here? */
11581	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11582	rc = VERR_EM_INTERPRETER;
11583	break;
11584	}
11585
11586	switch (uVector)
11587	{
11588	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11589	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11590	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11591	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11592	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11593	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11594	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11595
11596	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11597	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11598	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11599	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11600	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11601	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11602	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11603	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11604	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11605	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11606	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11607	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11608	default:
11609	{
11610	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11611	AssertRCReturn(rc, rc);
11612
11613	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11614	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11615	{
11616	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11617	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11618	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11619
11620	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11621	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11622	AssertRCReturn(rc, rc);
11623	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11624	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11625	0 /* GCPtrFaultAddress */);
11626	AssertRCReturn(rc, rc);
11627	}
11628	else
11629	{
11630	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11631	pVCpu->hm.s.u32HMError = uVector;
11632	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11633	}
11634	break;
11635	}
11636	}
11637	break;
11638	}
11639
11640	default:
11641	{
11642	pVCpu->hm.s.u32HMError = uExitIntInfo;
11643	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11644	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11645	break;
11646	}
11647	}
11648	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11649	return rc;
11650	}
11651
11652
11653	/**
11654	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11655	*/
11656	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11657	{
11658	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11659
11660	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11661	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11662
11663	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11665	return VINF_SUCCESS;
11666	}
11667
11668
11669	/**
11670	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11671	*/
11672	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11673	{
11674	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11675	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11676	{
11677	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11678	HMVMX_RETURN_UNEXPECTED_EXIT();
11679	}
11680
11681	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11682
11683	/*
11684	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11685	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11686	*/
11687	uint32_t uIntrState = 0;
11688	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11689	AssertRCReturn(rc, rc);
11690
11691	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11692	if ( fBlockSti
11693	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11694	{
11695	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11696	}
11697
11698	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11699	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11700
11701	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11702	return VINF_SUCCESS;
11703	}
11704
11705
11706	/**
11707	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11708	*/
11709	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11710	{
11711	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11712	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11713	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11714	}
11715
11716
11717	/**
11718	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11719	*/
11720	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11721	{
11722	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11724	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11725	}
11726
11727
11728	/**
11729	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11730	*/
11731	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11732	{
11733	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11734	PVM pVM = pVCpu->CTX_SUFF(pVM);
11735	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11736	if (RT_LIKELY(rc == VINF_SUCCESS))
11737	{
11738	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11739	Assert(pVmxTransient->cbInstr == 2);
11740	}
11741	else
11742	{
11743	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11744	rc = VERR_EM_INTERPRETER;
11745	}
11746	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11747	return rc;
11748	}
11749
11750
11751	/**
11752	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11753	*/
11754	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11755	{
11756	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11757	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11758	AssertRCReturn(rc, rc);
11759
11760	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11761	return VINF_EM_RAW_EMULATE_INSTR;
11762
11763	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11764	HMVMX_RETURN_UNEXPECTED_EXIT();
11765	}
11766
11767
11768	/**
11769	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11770	*/
11771	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11772	{
11773	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11774	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11775	AssertRCReturn(rc, rc);
11776
11777	PVM pVM = pVCpu->CTX_SUFF(pVM);
11778	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11779	if (RT_LIKELY(rc == VINF_SUCCESS))
11780	{
11781	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11782	Assert(pVmxTransient->cbInstr == 2);
11783	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11784	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11785	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11786	}
11787	else
11788	rc = VERR_EM_INTERPRETER;
11789	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11790	return rc;
11791	}
11792
11793
11794	/**
11795	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11796	*/
11797	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11798	{
11799	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11800	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11801	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11802	AssertRCReturn(rc, rc);
11803
11804	PVM pVM = pVCpu->CTX_SUFF(pVM);
11805	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11806	if (RT_SUCCESS(rc))
11807	{
11808	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11809	Assert(pVmxTransient->cbInstr == 3);
11810	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11811	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11812	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11813	}
11814	else
11815	{
11816	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11817	rc = VERR_EM_INTERPRETER;
11818	}
11819	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11820	return rc;
11821	}
11822
11823
11824	/**
11825	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11826	*/
11827	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11828	{
11829	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11830	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11831	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11832	AssertRCReturn(rc, rc);
11833
11834	PVM pVM = pVCpu->CTX_SUFF(pVM);
11835	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11836	if (RT_LIKELY(rc == VINF_SUCCESS))
11837	{
11838	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11839	Assert(pVmxTransient->cbInstr == 2);
11840	}
11841	else
11842	{
11843	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11844	rc = VERR_EM_INTERPRETER;
11845	}
11846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11847	return rc;
11848	}
11849
11850
11851	/**
11852	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11853	*/
11854	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11855	{
11856	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11858
11859	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11860	if (pVCpu->hm.s.fHypercallsEnabled)
11861	{
11862	#if 0
11863	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11864	#else
11865	/* Aggressive state sync. for now. */
11866	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11867	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11868	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11869	AssertRCReturn(rc, rc);
11870	#endif
11871
11872	/* Perform the hypercall. */
11873	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11874	if (rcStrict == VINF_SUCCESS)
11875	{
11876	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11877	AssertRCReturn(rc, rc);
11878	}
11879	else
11880	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11881	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11882	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11883
11884	/* If the hypercall changes anything other than guest's general-purpose registers,
11885	we would need to reload the guest changed bits here before VM-entry. */
11886	}
11887	else
11888	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11889
11890	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11891	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11892	{
11893	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11894	rcStrict = VINF_SUCCESS;
11895	}
11896
11897	return rcStrict;
11898	}
11899
11900
11901	/**
11902	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11903	*/
11904	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11905	{
11906	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11907	PVM pVM = pVCpu->CTX_SUFF(pVM);
11908	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11909
11910	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11911	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11912	AssertRCReturn(rc, rc);
11913
11914	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11915	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11916	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11917	else
11918	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11919	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11921	return rcStrict;
11922	}
11923
11924
11925	/**
11926	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11927	*/
11928	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11929	{
11930	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11931	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11932	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11933	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11934	AssertRCReturn(rc, rc);
11935
11936	PVM pVM = pVCpu->CTX_SUFF(pVM);
11937	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11938	if (RT_LIKELY(rc == VINF_SUCCESS))
11939	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11940	else
11941	{
11942	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11943	rc = VERR_EM_INTERPRETER;
11944	}
11945	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11946	return rc;
11947	}
11948
11949
11950	/**
11951	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11952	*/
11953	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11954	{
11955	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11956	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11957	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11958	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11959	AssertRCReturn(rc, rc);
11960
11961	PVM pVM = pVCpu->CTX_SUFF(pVM);
11962	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11963	rc = VBOXSTRICTRC_VAL(rc2);
11964	if (RT_LIKELY( rc == VINF_SUCCESS
11965	\|\| rc == VINF_EM_HALT))
11966	{
11967	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11968	AssertRCReturn(rc3, rc3);
11969
11970	if ( rc == VINF_EM_HALT
11971	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11972	{
11973	rc = VINF_SUCCESS;
11974	}
11975	}
11976	else
11977	{
11978	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11979	rc = VERR_EM_INTERPRETER;
11980	}
11981	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11982	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11984	return rc;
11985	}
11986
11987
11988	/**
11989	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11990	*/
11991	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11992	{
11993	/*
11994	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11995	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11996	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
11997	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11998	*/
11999	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12000	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12001	HMVMX_RETURN_UNEXPECTED_EXIT();
12002	}
12003
12004
12005	/**
12006	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12007	*/
12008	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12009	{
12010	/*
12011	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12012	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12013	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12014	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12015	*/
12016	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12017	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12018	HMVMX_RETURN_UNEXPECTED_EXIT();
12019	}
12020
12021
12022	/**
12023	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12024	*/
12025	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12026	{
12027	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12028	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12029	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12030	HMVMX_RETURN_UNEXPECTED_EXIT();
12031	}
12032
12033
12034	/**
12035	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12036	*/
12037	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12038	{
12039	/*
12040	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12041	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12042	* See Intel spec. 25.3 "Other Causes of VM-exits".
12043	*/
12044	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12045	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12046	HMVMX_RETURN_UNEXPECTED_EXIT();
12047	}
12048
12049
12050	/**
12051	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12052	* VM-exit.
12053	*/
12054	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12055	{
12056	/*
12057	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12058	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12059	*
12060	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12061	* See Intel spec. "23.8 Restrictions on VMX operation".
12062	*/
12063	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12064	return VINF_SUCCESS;
12065	}
12066
12067
12068	/**
12069	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12070	* VM-exit.
12071	*/
12072	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12073	{
12074	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12075	return VINF_EM_RESET;
12076	}
12077
12078
12079	/**
12080	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12081	*/
12082	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12083	{
12084	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12085	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12086
12087	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12088	AssertRCReturn(rc, rc);
12089
12090	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12091	rc = VINF_SUCCESS;
12092	else
12093	rc = VINF_EM_HALT;
12094
12095	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12096	if (rc != VINF_SUCCESS)
12097	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12098	return rc;
12099	}
12100
12101
12102	/**
12103	* VM-exit handler for instructions that result in a \#UD exception delivered to
12104	* the guest.
12105	*/
12106	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12107	{
12108	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12109	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12110	return VINF_SUCCESS;
12111	}
12112
12113
12114	/**
12115	* VM-exit handler for expiry of the VMX preemption timer.
12116	*/
12117	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12118	{
12119	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12120
12121	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12122	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12123
12124	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12125	PVM pVM = pVCpu->CTX_SUFF(pVM);
12126	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12127	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12128	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12129	}
12130
12131
12132	/**
12133	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12134	*/
12135	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12136	{
12137	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12138
12139	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12140	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12141	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12142	AssertRCReturn(rc, rc);
12143
12144	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12145	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12146
12147	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12148
12149	return rcStrict;
12150	}
12151
12152
12153	/**
12154	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12155	*/
12156	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12157	{
12158	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12159
12160	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
12161	/** @todo implement EMInterpretInvpcid() */
12162	return VERR_EM_INTERPRETER;
12163	}
12164
12165
12166	/**
12167	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12168	* Error VM-exit.
12169	*/
12170	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12171	{
12172	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12173	AssertRCReturn(rc, rc);
12174
12175	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12176	AssertRCReturn(rc, rc);
12177
12178	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12179	NOREF(uInvalidReason);
12180
12181	#ifdef VBOX_STRICT
12182	uint32_t uIntrState;
12183	RTHCUINTREG uHCReg;
12184	uint64_t u64Val;
12185	uint32_t u32Val;
12186
12187	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12188	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12189	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12190	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12191	AssertRCReturn(rc, rc);
12192
12193	Log4(("uInvalidReason %u\n", uInvalidReason));
12194	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12195	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12196	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12197	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12198
12199	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12200	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12201	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12202	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12203	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12204	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12205	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12206	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12207	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12208	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12209	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12210	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12211	#else
12212	NOREF(pVmxTransient);
12213	#endif
12214
12215	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12216	return VERR_VMX_INVALID_GUEST_STATE;
12217	}
12218
12219
12220	/**
12221	* VM-exit handler for VM-entry failure due to an MSR-load
12222	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12223	*/
12224	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12225	{
12226	NOREF(pVmxTransient);
12227	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12228	HMVMX_RETURN_UNEXPECTED_EXIT();
12229	}
12230
12231
12232	/**
12233	* VM-exit handler for VM-entry failure due to a machine-check event
12234	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12235	*/
12236	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12237	{
12238	NOREF(pVmxTransient);
12239	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12240	HMVMX_RETURN_UNEXPECTED_EXIT();
12241	}
12242
12243
12244	/**
12245	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12246	* theory.
12247	*/
12248	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12249	{
12250	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12251	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12252	return VERR_VMX_UNDEFINED_EXIT_CODE;
12253	}
12254
12255
12256	/**
12257	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12258	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12259	* Conditional VM-exit.
12260	*/
12261	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12262	{
12263	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12264
12265	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12267	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12268	return VERR_EM_INTERPRETER;
12269	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12270	HMVMX_RETURN_UNEXPECTED_EXIT();
12271	}
12272
12273
12274	/**
12275	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12276	*/
12277	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12278	{
12279	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12280
12281	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12283	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12284	return VERR_EM_INTERPRETER;
12285	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12286	HMVMX_RETURN_UNEXPECTED_EXIT();
12287	}
12288
12289
12290	/**
12291	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12292	*/
12293	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12294	{
12295	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12296
12297	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12298	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12299	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12300	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12301	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12302	{
12303	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12304	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12305	}
12306	AssertRCReturn(rc, rc);
12307	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12308
12309	#ifdef VBOX_STRICT
12310	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12311	{
12312	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12313	&& pMixedCtx->ecx != MSR_K6_EFER)
12314	{
12315	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12316	pMixedCtx->ecx));
12317	HMVMX_RETURN_UNEXPECTED_EXIT();
12318	}
12319	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12320	{
12321	VMXMSREXITREAD enmRead;
12322	VMXMSREXITWRITE enmWrite;
12323	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12324	AssertRCReturn(rc2, rc2);
12325	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12326	{
12327	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12328	HMVMX_RETURN_UNEXPECTED_EXIT();
12329	}
12330	}
12331	}
12332	#endif
12333
12334	PVM pVM = pVCpu->CTX_SUFF(pVM);
12335	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12336	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12337	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12339	if (RT_SUCCESS(rc))
12340	{
12341	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12342	Assert(pVmxTransient->cbInstr == 2);
12343	}
12344	return rc;
12345	}
12346
12347
12348	/**
12349	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12350	*/
12351	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12352	{
12353	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12354	PVM pVM = pVCpu->CTX_SUFF(pVM);
12355	int rc = VINF_SUCCESS;
12356
12357	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12358	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12359	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12360	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12361	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12362	{
12363	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12364	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12365	}
12366	AssertRCReturn(rc, rc);
12367	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12368
12369	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12370	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12371	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12372
12373	if (RT_SUCCESS(rc))
12374	{
12375	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12376
12377	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12378	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12379	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12380	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12381	{
12382	/*
12383	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12384	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12385	* EMInterpretWrmsr() changes it.
12386	*/
12387	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12388	}
12389	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12390	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12391	else if (pMixedCtx->ecx == MSR_K6_EFER)
12392	{
12393	/*
12394	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12395	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12396	* the other bits as well, SCE and NXE. See @bugref{7368}.
12397	*/
12398	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12399	}
12400
12401	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12402	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12403	{
12404	switch (pMixedCtx->ecx)
12405	{
12406	/*
12407	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12408	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12409	*/
12410	case MSR_IA32_SYSENTER_CS:
12411	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12412	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12413	break;
12414	case MSR_IA32_SYSENTER_EIP:
12415	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12416	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12417	break;
12418	case MSR_IA32_SYSENTER_ESP:
12419	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12420	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12421	break;
12422	case MSR_K8_FS_BASE: /* fall thru */
12423	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12424	case MSR_K6_EFER: /* already handled above */ break;
12425	default:
12426	{
12427	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12428	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12429	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12430	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12431	break;
12432	}
12433	}
12434	}
12435	#ifdef VBOX_STRICT
12436	else
12437	{
12438	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12439	switch (pMixedCtx->ecx)
12440	{
12441	case MSR_IA32_SYSENTER_CS:
12442	case MSR_IA32_SYSENTER_EIP:
12443	case MSR_IA32_SYSENTER_ESP:
12444	case MSR_K8_FS_BASE:
12445	case MSR_K8_GS_BASE:
12446	{
12447	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12448	HMVMX_RETURN_UNEXPECTED_EXIT();
12449	}
12450
12451	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12452	default:
12453	{
12454	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12455	{
12456	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12457	if (pMixedCtx->ecx != MSR_K6_EFER)
12458	{
12459	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12460	pMixedCtx->ecx));
12461	HMVMX_RETURN_UNEXPECTED_EXIT();
12462	}
12463	}
12464
12465	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12466	{
12467	VMXMSREXITREAD enmRead;
12468	VMXMSREXITWRITE enmWrite;
12469	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12470	AssertRCReturn(rc2, rc2);
12471	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12472	{
12473	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12474	HMVMX_RETURN_UNEXPECTED_EXIT();
12475	}
12476	}
12477	break;
12478	}
12479	}
12480	}
12481	#endif /* VBOX_STRICT */
12482	}
12483	return rc;
12484	}
12485
12486
12487	/**
12488	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12489	*/
12490	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12491	{
12492	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12493
12494	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12495	return VINF_EM_RAW_INTERRUPT;
12496	}
12497
12498
12499	/**
12500	* VM-exit handler for when the TPR value is lowered below the specified
12501	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12502	*/
12503	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12504	{
12505	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12506	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12507
12508	/*
12509	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12510	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12511	*/
12512	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12513	return VINF_SUCCESS;
12514	}
12515
12516
12517	/**
12518	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12519	* VM-exit.
12520	*
12521	* @retval VINF_SUCCESS when guest execution can continue.
12522	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12523	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12524	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12525	* interpreter.
12526	*/
12527	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12528	{
12529	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12530	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12531	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12532	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12533	AssertRCReturn(rc, rc);
12534
12535	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12536	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12537	PVM pVM = pVCpu->CTX_SUFF(pVM);
12538	VBOXSTRICTRC rcStrict;
12539	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12540	switch (uAccessType)
12541	{
12542	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12543	{
12544	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12545	AssertRCReturn(rc, rc);
12546
12547	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12548	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12549	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12550	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12551	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12552	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12553	{
12554	case 0: /* CR0 */
12555	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12556	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12557	break;
12558	case 2: /* CR2 */
12559	/* Nothing to do here, CR2 it's not part of the VMCS. */
12560	break;
12561	case 3: /* CR3 */
12562	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12563	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12564	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12565	break;
12566	case 4: /* CR4 */
12567	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12568	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12569	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12570	break;
12571	case 8: /* CR8 */
12572	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12573	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12574	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12575	break;
12576	default:
12577	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12578	break;
12579	}
12580
12581	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12582	break;
12583	}
12584
12585	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12586	{
12587	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12588	AssertRCReturn(rc, rc);
12589
12590	Assert( !pVM->hm.s.fNestedPaging
12591	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12592	\|\| pVCpu->hm.s.fUsingDebugLoop
12593	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12594
12595	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12596	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12597	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12598
12599	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12600	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12601	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12602	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12603	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12604	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12605	VBOXSTRICTRC_VAL(rcStrict)));
12606	break;
12607	}
12608
12609	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12610	{
12611	AssertRCReturn(rc, rc);
12612	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12613	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12614	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12615	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12616	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12617	break;
12618	}
12619
12620	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12621	{
12622	AssertRCReturn(rc, rc);
12623	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12624	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12625	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12626	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12627	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12629	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12630	break;
12631	}
12632
12633	default:
12634	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12635	VERR_VMX_UNEXPECTED_EXCEPTION);
12636	}
12637
12638	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12639	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12640	NOREF(pVM);
12641	return rcStrict;
12642	}
12643
12644
12645	/**
12646	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12647	* VM-exit.
12648	*/
12649	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12650	{
12651	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12652	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12653
12654	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12655	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12656	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12657	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12658	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12659	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12660	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12661	AssertRCReturn(rc, rc);
12662
12663	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12664	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12665	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12666	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12667	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12668	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12669	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12670	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12671	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12672
12673	/* I/O operation lookup arrays. */
12674	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12675	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12676
12677	VBOXSTRICTRC rcStrict;
12678	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12679	uint32_t const cbInstr = pVmxTransient->cbInstr;
12680	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12681	PVM pVM = pVCpu->CTX_SUFF(pVM);
12682	if (fIOString)
12683	{
12684	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12685	See @bugref{5752#c158}. Should work now. */
12686	/*
12687	* INS/OUTS - I/O String instruction.
12688	*
12689	* Use instruction-information if available, otherwise fall back on
12690	* interpreting the instruction.
12691	*/
12692	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12693	fIOWrite ? 'w' : 'r'));
12694	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12695	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12696	{
12697	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12698	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12699	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12700	AssertRCReturn(rc2, rc2);
12701	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12702	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12703	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12704	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12705	if (fIOWrite)
12706	{
12707	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12708	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12709	}
12710	else
12711	{
12712	/*
12713	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12714	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12715	* See Intel Instruction spec. for "INS".
12716	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12717	*/
12718	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12719	}
12720	}
12721	else
12722	{
12723	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12724	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12725	AssertRCReturn(rc2, rc2);
12726	rcStrict = IEMExecOne(pVCpu);
12727	}
12728	/** @todo IEM needs to be setting these flags somehow. */
12729	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12730	fUpdateRipAlready = true;
12731	#else
12732	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12733	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12734	if (RT_SUCCESS(rcStrict))
12735	{
12736	if (fIOWrite)
12737	{
12738	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12739	(DISCPUMODE)pDis->uAddrMode, cbValue);
12740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12741	}
12742	else
12743	{
12744	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12745	(DISCPUMODE)pDis->uAddrMode, cbValue);
12746	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12747	}
12748	}
12749	else
12750	{
12751	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12752	pMixedCtx->rip));
12753	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12754	}
12755	#endif
12756	}
12757	else
12758	{
12759	/*
12760	* IN/OUT - I/O instruction.
12761	*/
12762	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12763	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12764	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12765	if (fIOWrite)
12766	{
12767	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12768	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12769	}
12770	else
12771	{
12772	uint32_t u32Result = 0;
12773	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12774	if (IOM_SUCCESS(rcStrict))
12775	{
12776	/* Save result of I/O IN instr. in AL/AX/EAX. */
12777	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12778	}
12779	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12780	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12782	}
12783	}
12784
12785	if (IOM_SUCCESS(rcStrict))
12786	{
12787	if (!fUpdateRipAlready)
12788	{
12789	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12790	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12791	}
12792
12793	/*
12794	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12795	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12796	*/
12797	if (fIOString)
12798	{
12799	/** @todo Single-step for INS/OUTS with REP prefix? */
12800	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12801	}
12802	else if ( !fDbgStepping
12803	&& fGstStepping)
12804	{
12805	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12806	}
12807
12808	/*
12809	* If any I/O breakpoints are armed, we need to check if one triggered
12810	* and take appropriate action.
12811	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12812	*/
12813	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12814	AssertRCReturn(rc2, rc2);
12815
12816	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12817	* execution engines about whether hyper BPs and such are pending. */
12818	uint32_t const uDr7 = pMixedCtx->dr[7];
12819	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12820	&& X86_DR7_ANY_RW_IO(uDr7)
12821	&& (pMixedCtx->cr4 & X86_CR4_DE))
12822	\|\| DBGFBpIsHwIoArmed(pVM)))
12823	{
12824	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12825
12826	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12827	VMMRZCallRing3Disable(pVCpu);
12828	HM_DISABLE_PREEMPT();
12829
12830	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12831
12832	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12833	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12834	{
12835	/* Raise #DB. */
12836	if (fIsGuestDbgActive)
12837	ASMSetDR6(pMixedCtx->dr[6]);
12838	if (pMixedCtx->dr[7] != uDr7)
12839	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12840
12841	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12842	}
12843	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12844	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12845	else if ( rcStrict2 != VINF_SUCCESS
12846	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12847	rcStrict = rcStrict2;
12848	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12849
12850	HM_RESTORE_PREEMPT();
12851	VMMRZCallRing3Enable(pVCpu);
12852	}
12853	}
12854
12855	#ifdef VBOX_STRICT
12856	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12857	Assert(!fIOWrite);
12858	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12859	Assert(fIOWrite);
12860	else
12861	{
12862	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12863	* statuses, that the VMM device and some others may return. See
12864	* IOM_SUCCESS() for guidance. */
12865	AssertMsg( RT_FAILURE(rcStrict)
12866	\|\| rcStrict == VINF_SUCCESS
12867	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12868	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12869	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12870	\|\| rcStrict == VINF_EM_RAW_TO_R3
12871	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12872	#endif
12873	}
12874	#endif
12875
12876	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12877	return rcStrict;
12878	}
12879
12880
12881	/**
12882	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12883	* VM-exit.
12884	*/
12885	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12886	{
12887	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12888
12889	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12890	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12891	AssertRCReturn(rc, rc);
12892	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12893	{
12894	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12895	AssertRCReturn(rc, rc);
12896	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12897	{
12898	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12899
12900	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12901	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12902
12903	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12904	Assert(!pVCpu->hm.s.Event.fPending);
12905	pVCpu->hm.s.Event.fPending = true;
12906	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12907	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12908	AssertRCReturn(rc, rc);
12909	if (fErrorCodeValid)
12910	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12911	else
12912	pVCpu->hm.s.Event.u32ErrCode = 0;
12913	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12914	&& uVector == X86_XCPT_PF)
12915	{
12916	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12917	}
12918
12919	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12921	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12922	}
12923	}
12924
12925	/* Fall back to the interpreter to emulate the task-switch. */
12926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12927	return VERR_EM_INTERPRETER;
12928	}
12929
12930
12931	/**
12932	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12933	*/
12934	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12935	{
12936	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12937	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12938	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12939	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12940	AssertRCReturn(rc, rc);
12941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12942	return VINF_EM_DBG_STEPPED;
12943	}
12944
12945
12946	/**
12947	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12948	*/
12949	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12950	{
12951	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12952
12953	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12954
12955	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12956	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12957	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12958	{
12959	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12960	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12961	{
12962	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12963	return VERR_EM_INTERPRETER;
12964	}
12965	}
12966	else
12967	{
12968	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12969	rcStrict1 = VINF_SUCCESS;
12970	return rcStrict1;
12971	}
12972
12973	#if 0
12974	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12975	* just sync the whole thing. */
12976	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12977	#else
12978	/* Aggressive state sync. for now. */
12979	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12980	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12981	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12982	#endif
12983	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12984	AssertRCReturn(rc, rc);
12985
12986	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12987	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12988	VBOXSTRICTRC rcStrict2;
12989	switch (uAccessType)
12990	{
12991	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12992	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12993	{
12994	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12995	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12996	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12997
12998	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12999	GCPhys &= PAGE_BASE_GC_MASK;
13000	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13001	PVM pVM = pVCpu->CTX_SUFF(pVM);
13002	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13003	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13004
13005	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13006	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13007	CPUMCTX2CORE(pMixedCtx), GCPhys);
13008	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13009	if ( rcStrict2 == VINF_SUCCESS
13010	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13011	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13012	{
13013	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13014	\| HM_CHANGED_GUEST_RSP
13015	\| HM_CHANGED_GUEST_RFLAGS
13016	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13017	rcStrict2 = VINF_SUCCESS;
13018	}
13019	break;
13020	}
13021
13022	default:
13023	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13024	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13025	break;
13026	}
13027
13028	if (rcStrict2 != VINF_SUCCESS)
13029	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13030	return rcStrict2;
13031	}
13032
13033
13034	/**
13035	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13036	* VM-exit.
13037	*/
13038	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13039	{
13040	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13041
13042	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13043	if (pVmxTransient->fWasGuestDebugStateActive)
13044	{
13045	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13046	HMVMX_RETURN_UNEXPECTED_EXIT();
13047	}
13048
13049	if ( !pVCpu->hm.s.fSingleInstruction
13050	&& !pVmxTransient->fWasHyperDebugStateActive)
13051	{
13052	Assert(!DBGFIsStepping(pVCpu));
13053	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13054
13055	/* Don't intercept MOV DRx any more. */
13056	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13057	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13058	AssertRCReturn(rc, rc);
13059
13060	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13061	VMMRZCallRing3Disable(pVCpu);
13062	HM_DISABLE_PREEMPT();
13063
13064	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13065	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13066	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13067
13068	HM_RESTORE_PREEMPT();
13069	VMMRZCallRing3Enable(pVCpu);
13070
13071	#ifdef VBOX_WITH_STATISTICS
13072	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13073	AssertRCReturn(rc, rc);
13074	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13075	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13076	else
13077	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13078	#endif
13079	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13080	return VINF_SUCCESS;
13081	}
13082
13083	/*
13084	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13085	* Update the segment registers and DR7 from the CPU.
13086	*/
13087	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13088	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13089	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13090	AssertRCReturn(rc, rc);
13091	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13092
13093	PVM pVM = pVCpu->CTX_SUFF(pVM);
13094	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13095	{
13096	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13097	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13098	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13099	if (RT_SUCCESS(rc))
13100	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13102	}
13103	else
13104	{
13105	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13106	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13107	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13108	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13109	}
13110
13111	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13112	if (RT_SUCCESS(rc))
13113	{
13114	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13115	AssertRCReturn(rc2, rc2);
13116	return VINF_SUCCESS;
13117	}
13118	return rc;
13119	}
13120
13121
13122	/**
13123	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13124	* Conditional VM-exit.
13125	*/
13126	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13127	{
13128	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13129	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13130
13131	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13132	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13133	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13134	{
13135	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13136	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13137	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13138	{
13139	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13140	return VERR_EM_INTERPRETER;
13141	}
13142	}
13143	else
13144	{
13145	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13146	rcStrict1 = VINF_SUCCESS;
13147	return rcStrict1;
13148	}
13149
13150	RTGCPHYS GCPhys = 0;
13151	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13152
13153	#if 0
13154	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13155	#else
13156	/* Aggressive state sync. for now. */
13157	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13158	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13159	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13160	#endif
13161	AssertRCReturn(rc, rc);
13162
13163	/*
13164	* If we succeed, resume guest execution.
13165	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13166	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13167	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13168	* weird case. See @bugref{6043}.
13169	*/
13170	PVM pVM = pVCpu->CTX_SUFF(pVM);
13171	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13172	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13173	if ( rcStrict2 == VINF_SUCCESS
13174	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13175	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13176	{
13177	/* Successfully handled MMIO operation. */
13178	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13179	\| HM_CHANGED_GUEST_RSP
13180	\| HM_CHANGED_GUEST_RFLAGS
13181	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13182	return VINF_SUCCESS;
13183	}
13184	return rcStrict2;
13185	}
13186
13187
13188	/**
13189	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13190	* VM-exit.
13191	*/
13192	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13193	{
13194	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13195	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13196
13197	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13198	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13199	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13200	{
13201	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13202	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13203	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13204	}
13205	else
13206	{
13207	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13208	rcStrict1 = VINF_SUCCESS;
13209	return rcStrict1;
13210	}
13211
13212	RTGCPHYS GCPhys = 0;
13213	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13214	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13215	#if 0
13216	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13217	#else
13218	/* Aggressive state sync. for now. */
13219	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13220	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13221	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13222	#endif
13223	AssertRCReturn(rc, rc);
13224
13225	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13226	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13227
13228	RTGCUINT uErrorCode = 0;
13229	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13230	uErrorCode \|= X86_TRAP_PF_ID;
13231	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13232	uErrorCode \|= X86_TRAP_PF_RW;
13233	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13234	uErrorCode \|= X86_TRAP_PF_P;
13235
13236	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13237
13238	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13239	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13240
13241	/* Handle the pagefault trap for the nested shadow table. */
13242	PVM pVM = pVCpu->CTX_SUFF(pVM);
13243	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13244	TRPMResetTrap(pVCpu);
13245
13246	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13247	if ( rcStrict2 == VINF_SUCCESS
13248	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13249	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13250	{
13251	/* Successfully synced our nested page tables. */
13252	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13253	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13254	\| HM_CHANGED_GUEST_RSP
13255	\| HM_CHANGED_GUEST_RFLAGS);
13256	return VINF_SUCCESS;
13257	}
13258
13259	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13260	return rcStrict2;
13261	}
13262
13263	/** @} */
13264
13265	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13266	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13267	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13268
13269	/** @name VM-exit exception handlers.
13270	* @{
13271	*/
13272
13273	/**
13274	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13275	*/
13276	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13277	{
13278	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13280
13281	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13282	AssertRCReturn(rc, rc);
13283
13284	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13285	{
13286	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13287	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13288
13289	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13290	* provides VM-exit instruction length. If this causes problem later,
13291	* disassemble the instruction like it's done on AMD-V. */
13292	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13293	AssertRCReturn(rc2, rc2);
13294	return rc;
13295	}
13296
13297	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13298	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13299	return rc;
13300	}
13301
13302
13303	/**
13304	* VM-exit exception handler for \#BP (Breakpoint exception).
13305	*/
13306	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13307	{
13308	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13309	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13310
13311	/** @todo Try optimize this by not saving the entire guest state unless
13312	* really needed. */
13313	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13314	AssertRCReturn(rc, rc);
13315
13316	PVM pVM = pVCpu->CTX_SUFF(pVM);
13317	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13318	if (rc == VINF_EM_RAW_GUEST_TRAP)
13319	{
13320	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13321	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13322	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13323	AssertRCReturn(rc, rc);
13324
13325	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13326	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13327	}
13328
13329	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13330	return rc;
13331	}
13332
13333
13334	/**
13335	* VM-exit exception handler for \#AC (alignment check exception).
13336	*/
13337	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13338	{
13339	RT_NOREF_PV(pMixedCtx);
13340	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13341
13342	/*
13343	* Re-inject it. We'll detect any nesting before getting here.
13344	*/
13345	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13346	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13347	AssertRCReturn(rc, rc);
13348	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13349
13350	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13351	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13352	return VINF_SUCCESS;
13353	}
13354
13355
13356	/**
13357	* VM-exit exception handler for \#DB (Debug exception).
13358	*/
13359	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13360	{
13361	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13362	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13363	Log6(("XcptDB\n"));
13364
13365	/*
13366	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13367	* for processing.
13368	*/
13369	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13370	AssertRCReturn(rc, rc);
13371
13372	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13373	uint64_t uDR6 = X86_DR6_INIT_VAL;
13374	uDR6 \|= ( pVmxTransient->uExitQualification
13375	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13376
13377	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13378	if (rc == VINF_EM_RAW_GUEST_TRAP)
13379	{
13380	/*
13381	* The exception was for the guest. Update DR6, DR7.GD and
13382	* IA32_DEBUGCTL.LBR before forwarding it.
13383	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13384	*/
13385	VMMRZCallRing3Disable(pVCpu);
13386	HM_DISABLE_PREEMPT();
13387
13388	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13389	pMixedCtx->dr[6] \|= uDR6;
13390	if (CPUMIsGuestDebugStateActive(pVCpu))
13391	ASMSetDR6(pMixedCtx->dr[6]);
13392
13393	HM_RESTORE_PREEMPT();
13394	VMMRZCallRing3Enable(pVCpu);
13395
13396	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13397	AssertRCReturn(rc, rc);
13398
13399	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13400	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13401
13402	/* Paranoia. */
13403	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13404	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13405
13406	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13407	AssertRCReturn(rc, rc);
13408
13409	/*
13410	* Raise #DB in the guest.
13411	*
13412	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13413	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13414	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13415	*
13416	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13417	*/
13418	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13419	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13420	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13421	AssertRCReturn(rc, rc);
13422	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13423	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13424	return VINF_SUCCESS;
13425	}
13426
13427	/*
13428	* Not a guest trap, must be a hypervisor related debug event then.
13429	* Update DR6 in case someone is interested in it.
13430	*/
13431	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13432	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13433	CPUMSetHyperDR6(pVCpu, uDR6);
13434
13435	return rc;
13436	}
13437
13438
13439	/**
13440	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13441	* point exception).
13442	*/
13443	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13444	{
13445	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13446
13447	/* We require CR0 and EFER. EFER is always up-to-date. */
13448	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13449	AssertRCReturn(rc, rc);
13450
13451	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13452	VMMRZCallRing3Disable(pVCpu);
13453	HM_DISABLE_PREEMPT();
13454
13455	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13456	if (pVmxTransient->fWasGuestFPUStateActive)
13457	{
13458	rc = VINF_EM_RAW_GUEST_TRAP;
13459	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13460	}
13461	else
13462	{
13463	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13464	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13465	#endif
13466	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13467	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13468	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13469	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13470	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13471	}
13472
13473	HM_RESTORE_PREEMPT();
13474	VMMRZCallRing3Enable(pVCpu);
13475
13476	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13477	{
13478	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13479	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13480	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13481	pVCpu->hm.s.fPreloadGuestFpu = true;
13482	}
13483	else
13484	{
13485	/* Forward #NM to the guest. */
13486	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13487	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13488	AssertRCReturn(rc, rc);
13489	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13490	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13491	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13492	}
13493
13494	return VINF_SUCCESS;
13495	}
13496
13497
13498	/**
13499	* VM-exit exception handler for \#GP (General-protection exception).
13500	*
13501	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13502	*/
13503	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13504	{
13505	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13507
13508	int rc;
13509	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13510	{ /* likely */ }
13511	else
13512	{
13513	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13514	Assert(pVCpu->hm.s.fUsingDebugLoop);
13515	#endif
13516	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13517	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13518	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13519	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13520	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13521	AssertRCReturn(rc, rc);
13522	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13523	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13524	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13525	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13526	return rc;
13527	}
13528
13529	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13530	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13531
13532	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13533	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13534	AssertRCReturn(rc, rc);
13535
13536	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13537	uint32_t cbOp = 0;
13538	PVM pVM = pVCpu->CTX_SUFF(pVM);
13539	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13540	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13541	if (RT_SUCCESS(rc))
13542	{
13543	rc = VINF_SUCCESS;
13544	Assert(cbOp == pDis->cbInstr);
13545	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13546	switch (pDis->pCurInstr->uOpcode)
13547	{
13548	case OP_CLI:
13549	{
13550	pMixedCtx->eflags.Bits.u1IF = 0;
13551	pMixedCtx->eflags.Bits.u1RF = 0;
13552	pMixedCtx->rip += pDis->cbInstr;
13553	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13554	if ( !fDbgStepping
13555	&& pMixedCtx->eflags.Bits.u1TF)
13556	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13557	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13558	break;
13559	}
13560
13561	case OP_STI:
13562	{
13563	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13564	pMixedCtx->eflags.Bits.u1IF = 1;
13565	pMixedCtx->eflags.Bits.u1RF = 0;
13566	pMixedCtx->rip += pDis->cbInstr;
13567	if (!fOldIF)
13568	{
13569	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13570	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13571	}
13572	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13573	if ( !fDbgStepping
13574	&& pMixedCtx->eflags.Bits.u1TF)
13575	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13576	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13577	break;
13578	}
13579
13580	case OP_HLT:
13581	{
13582	rc = VINF_EM_HALT;
13583	pMixedCtx->rip += pDis->cbInstr;
13584	pMixedCtx->eflags.Bits.u1RF = 0;
13585	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13586	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13587	break;
13588	}
13589
13590	case OP_POPF:
13591	{
13592	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13593	uint32_t cbParm;
13594	uint32_t uMask;
13595	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13596	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13597	{
13598	cbParm = 4;
13599	uMask = 0xffffffff;
13600	}
13601	else
13602	{
13603	cbParm = 2;
13604	uMask = 0xffff;
13605	}
13606
13607	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13608	RTGCPTR GCPtrStack = 0;
13609	X86EFLAGS Eflags;
13610	Eflags.u32 = 0;
13611	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13612	&GCPtrStack);
13613	if (RT_SUCCESS(rc))
13614	{
13615	Assert(sizeof(Eflags.u32) >= cbParm);
13616	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13617	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13618	}
13619	if (RT_FAILURE(rc))
13620	{
13621	rc = VERR_EM_INTERPRETER;
13622	break;
13623	}
13624	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13625	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13626	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13627	pMixedCtx->esp += cbParm;
13628	pMixedCtx->esp &= uMask;
13629	pMixedCtx->rip += pDis->cbInstr;
13630	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13631	\| HM_CHANGED_GUEST_RSP
13632	\| HM_CHANGED_GUEST_RFLAGS);
13633	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13634	POPF restores EFLAGS.TF. */
13635	if ( !fDbgStepping
13636	&& fGstStepping)
13637	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13638	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13639	break;
13640	}
13641
13642	case OP_PUSHF:
13643	{
13644	uint32_t cbParm;
13645	uint32_t uMask;
13646	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13647	{
13648	cbParm = 4;
13649	uMask = 0xffffffff;
13650	}
13651	else
13652	{
13653	cbParm = 2;
13654	uMask = 0xffff;
13655	}
13656
13657	/* Get the stack pointer & push the contents of eflags onto the stack. */
13658	RTGCPTR GCPtrStack = 0;
13659	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13660	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13661	if (RT_FAILURE(rc))
13662	{
13663	rc = VERR_EM_INTERPRETER;
13664	break;
13665	}
13666	X86EFLAGS Eflags = pMixedCtx->eflags;
13667	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13668	Eflags.Bits.u1RF = 0;
13669	Eflags.Bits.u1VM = 0;
13670
13671	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13672	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13673	{
13674	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13675	rc = VERR_EM_INTERPRETER;
13676	break;
13677	}
13678	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13679	pMixedCtx->esp -= cbParm;
13680	pMixedCtx->esp &= uMask;
13681	pMixedCtx->rip += pDis->cbInstr;
13682	pMixedCtx->eflags.Bits.u1RF = 0;
13683	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13684	\| HM_CHANGED_GUEST_RSP
13685	\| HM_CHANGED_GUEST_RFLAGS);
13686	if ( !fDbgStepping
13687	&& pMixedCtx->eflags.Bits.u1TF)
13688	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13689	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13690	break;
13691	}
13692
13693	case OP_IRET:
13694	{
13695	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13696	* instruction reference. */
13697	RTGCPTR GCPtrStack = 0;
13698	uint32_t uMask = 0xffff;
13699	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13700	uint16_t aIretFrame[3];
13701	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13702	{
13703	rc = VERR_EM_INTERPRETER;
13704	break;
13705	}
13706	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13707	&GCPtrStack);
13708	if (RT_SUCCESS(rc))
13709	{
13710	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13711	PGMACCESSORIGIN_HM));
13712	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13713	}
13714	if (RT_FAILURE(rc))
13715	{
13716	rc = VERR_EM_INTERPRETER;
13717	break;
13718	}
13719	pMixedCtx->eip = 0;
13720	pMixedCtx->ip = aIretFrame[0];
13721	pMixedCtx->cs.Sel = aIretFrame[1];
13722	pMixedCtx->cs.ValidSel = aIretFrame[1];
13723	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13724	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13725	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13726	pMixedCtx->sp += sizeof(aIretFrame);
13727	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13728	\| HM_CHANGED_GUEST_SEGMENT_REGS
13729	\| HM_CHANGED_GUEST_RSP
13730	\| HM_CHANGED_GUEST_RFLAGS);
13731	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13732	if ( !fDbgStepping
13733	&& fGstStepping)
13734	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13735	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13736	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13737	break;
13738	}
13739
13740	case OP_INT:
13741	{
13742	uint16_t uVector = pDis->Param1.uValue & 0xff;
13743	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13744	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13745	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13746	break;
13747	}
13748
13749	case OP_INTO:
13750	{
13751	if (pMixedCtx->eflags.Bits.u1OF)
13752	{
13753	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13754	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13755	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13756	}
13757	else
13758	{
13759	pMixedCtx->eflags.Bits.u1RF = 0;
13760	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13761	}
13762	break;
13763	}
13764
13765	default:
13766	{
13767	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13768	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13769	EMCODETYPE_SUPERVISOR);
13770	rc = VBOXSTRICTRC_VAL(rc2);
13771	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13772	/** @todo We have to set pending-debug exceptions here when the guest is
13773	* single-stepping depending on the instruction that was interpreted. */
13774	Log4(("#GP rc=%Rrc\n", rc));
13775	break;
13776	}
13777	}
13778	}
13779	else
13780	rc = VERR_EM_INTERPRETER;
13781
13782	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13783	("#GP Unexpected rc=%Rrc\n", rc));
13784	return rc;
13785	}
13786
13787
13788	/**
13789	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13790	* the exception reported in the VMX transient structure back into the VM.
13791	*
13792	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13793	* up-to-date.
13794	*/
13795	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13796	{
13797	RT_NOREF_PV(pMixedCtx);
13798	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13799	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13800	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13801	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13802	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13803	#endif
13804
13805	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13806	hmR0VmxCheckExitDueToEventDelivery(). */
13807	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13808	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13809	AssertRCReturn(rc, rc);
13810	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13811
13812	#ifdef DEBUG_ramshankar
13813	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13814	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13815	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13816	#endif
13817
13818	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13819	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13820	return VINF_SUCCESS;
13821	}
13822
13823
13824	/**
13825	* VM-exit exception handler for \#PF (Page-fault exception).
13826	*/
13827	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13828	{
13829	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13830	PVM pVM = pVCpu->CTX_SUFF(pVM);
13831	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13832	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13833	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13834	AssertRCReturn(rc, rc);
13835
13836	if (!pVM->hm.s.fNestedPaging)
13837	{ /* likely */ }
13838	else
13839	{
13840	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13841	Assert(pVCpu->hm.s.fUsingDebugLoop);
13842	#endif
13843	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13844	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13845	{
13846	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13847	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13848	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13849	}
13850	else
13851	{
13852	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13853	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13854	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13855	}
13856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13857	return rc;
13858	}
13859
13860	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13861	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13862	if (pVmxTransient->fVectoringPF)
13863	{
13864	Assert(pVCpu->hm.s.Event.fPending);
13865	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13866	}
13867
13868	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13869	AssertRCReturn(rc, rc);
13870
13871	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13872	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13873
13874	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13875	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13876	(RTGCPTR)pVmxTransient->uExitQualification);
13877
13878	Log4(("#PF: rc=%Rrc\n", rc));
13879	if (rc == VINF_SUCCESS)
13880	{
13881	#if 0
13882	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13883	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13884	* memory? We don't update the whole state here... */
13885	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13886	\| HM_CHANGED_GUEST_RSP
13887	\| HM_CHANGED_GUEST_RFLAGS
13888	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13889	#else
13890	/*
13891	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13892	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13893	*/
13894	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13895	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13896	#endif
13897	TRPMResetTrap(pVCpu);
13898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13899	return rc;
13900	}
13901
13902	if (rc == VINF_EM_RAW_GUEST_TRAP)
13903	{
13904	if (!pVmxTransient->fVectoringDoublePF)
13905	{
13906	/* It's a guest page fault and needs to be reflected to the guest. */
13907	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13908	TRPMResetTrap(pVCpu);
13909	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13910	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13911	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13912	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13913	}
13914	else
13915	{
13916	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13917	TRPMResetTrap(pVCpu);
13918	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13919	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13920	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13921	}
13922
13923	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13924	return VINF_SUCCESS;
13925	}
13926
13927	TRPMResetTrap(pVCpu);
13928	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13929	return rc;
13930	}
13931
13932	/** @} */
13933

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 66706

Download in other formats: